File size: 8,784 Bytes
1191b09
14db392
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
layer,module,loss,samples,damp,time
0,self_attn.k_proj,0.37272908,0.01000,0.963
0,self_attn.v_proj,0.06249057,0.01000,0.568
0,self_attn.q_proj,1.71414546,0.01000,0.582
0,self_attn.o_proj,0.27218719,0.01000,0.579
0,mlp.up_proj,1.97148761,0.01000,0.602
0,mlp.gate_proj,5.45231162,0.01000,0.596
0,mlp.down_proj,0.61297246,0.01000,3.775
1,self_attn.k_proj,0.27201448,0.01000,0.597
1,self_attn.v_proj,0.06198664,0.01000,0.593
1,self_attn.q_proj,0.98380148,0.01000,0.580
1,self_attn.o_proj,0.07505655,0.01000,0.641
1,mlp.up_proj,29.94800979,0.01000,0.609
1,mlp.gate_proj,46.14816320,0.01000,0.605
1,mlp.down_proj,1.18726570,0.01000,3.571
2,self_attn.k_proj,1.10431638,0.01000,0.573
2,self_attn.v_proj,0.17621659,0.01000,0.580
2,self_attn.q_proj,3.89487222,0.01000,0.582
2,self_attn.o_proj,0.19335656,0.01000,0.599
2,mlp.up_proj,39.82002804,0.01000,0.599
2,mlp.gate_proj,65.63606855,0.01000,0.596
2,mlp.down_proj,2.19458135,0.01000,3.704
3,self_attn.k_proj,1.41276497,0.01000,0.583
3,self_attn.v_proj,0.32184546,0.01000,0.565
3,self_attn.q_proj,5.03698810,0.01000,0.568
3,self_attn.o_proj,0.52726446,0.01000,0.599
3,mlp.up_proj,114.33669945,0.01000,0.638
3,mlp.gate_proj,148.85794254,0.01000,0.621
3,mlp.down_proj,0.01009648,0.01250,4.438
4,self_attn.k_proj,2.52166661,0.01000,0.639
4,self_attn.v_proj,0.67622127,0.01000,0.620
4,self_attn.q_proj,10.74245164,0.01000,0.600
4,self_attn.o_proj,0.42747642,0.01000,0.602
4,mlp.up_proj,110.46822172,0.01000,0.661
4,mlp.gate_proj,161.16622643,0.01000,0.668
4,mlp.down_proj,2.72548815,0.01000,3.644
5,self_attn.k_proj,2.36542153,0.01000,0.596
5,self_attn.v_proj,0.82987875,0.01000,0.580
5,self_attn.q_proj,10.98670529,0.01000,0.583
5,self_attn.o_proj,0.48163516,0.01000,0.578
5,mlp.up_proj,169.33053275,0.01000,0.606
5,mlp.gate_proj,208.03480575,0.01000,0.632
5,mlp.down_proj,2.63673970,0.01000,3.785
6,self_attn.k_proj,1.51797450,0.01000,0.567
6,self_attn.v_proj,0.70319183,0.01000,0.574
6,self_attn.q_proj,7.39946994,0.01000,0.575
6,self_attn.o_proj,0.58445273,0.01000,0.597
6,mlp.up_proj,40.56810539,0.01000,0.589
6,mlp.gate_proj,57.70911391,0.01000,0.594
6,mlp.down_proj,4.15842980,0.01000,3.864
7,self_attn.k_proj,1.65347762,0.01000,0.605
7,self_attn.v_proj,1.26919441,0.01000,0.621
7,self_attn.q_proj,8.91557116,0.01000,0.617
7,self_attn.o_proj,1.26652386,0.01000,0.634
7,mlp.up_proj,37.50803346,0.01000,0.644
7,mlp.gate_proj,41.65068270,0.01000,0.663
7,mlp.down_proj,5.51070028,0.01000,3.738
8,self_attn.k_proj,3.05815341,0.01000,0.599
8,self_attn.v_proj,1.05918791,0.01000,0.605
8,self_attn.q_proj,13.26367949,0.01000,0.607
8,self_attn.o_proj,1.30098944,0.01000,0.605
8,mlp.up_proj,39.01938902,0.01000,0.591
8,mlp.gate_proj,40.47557959,0.01000,0.582
8,mlp.down_proj,5.03287961,0.01000,3.769
9,self_attn.k_proj,2.22347360,0.01000,0.609
9,self_attn.v_proj,1.48193391,0.01000,0.608
9,self_attn.q_proj,11.94389274,0.01000,0.585
9,self_attn.o_proj,1.67733308,0.01000,0.602
9,mlp.up_proj,72.89947883,0.01000,0.633
9,mlp.gate_proj,116.10071928,0.01000,0.614
9,mlp.down_proj,4.55470750,0.01000,3.589
10,self_attn.k_proj,2.02291953,0.01000,0.587
10,self_attn.v_proj,0.91743158,0.01000,0.572
10,self_attn.q_proj,10.23927115,0.01000,0.586
10,self_attn.o_proj,1.26303708,0.01000,0.628
10,mlp.up_proj,38.33327576,0.01000,0.631
10,mlp.gate_proj,42.22324041,0.01000,0.631
10,mlp.down_proj,4.19643478,0.01000,3.880
11,self_attn.k_proj,2.49720556,0.01000,0.604
11,self_attn.v_proj,0.84860099,0.01000,0.604
11,self_attn.q_proj,10.59318743,0.01000,0.614
11,self_attn.o_proj,1.44926018,0.01000,0.613
11,mlp.up_proj,34.05972397,0.01000,0.600
11,mlp.gate_proj,34.98881461,0.01000,0.621
11,mlp.down_proj,3.78504640,0.01000,3.701
12,self_attn.k_proj,2.51501925,0.01000,0.643
12,self_attn.v_proj,1.06571939,0.01000,0.612
12,self_attn.q_proj,11.37260368,0.01000,0.595
12,self_attn.o_proj,1.42345318,0.01000,0.574
12,mlp.up_proj,33.02424516,0.01000,0.609
12,mlp.gate_proj,32.15129480,0.01000,0.631
12,mlp.down_proj,4.32310599,0.01000,3.762
13,self_attn.k_proj,2.17874396,0.01000,0.600
13,self_attn.v_proj,1.34991701,0.01000,0.585
13,self_attn.q_proj,11.63673653,0.01000,0.595
13,self_attn.o_proj,2.12498794,0.01000,0.577
13,mlp.up_proj,31.13961415,0.01000,0.606
13,mlp.gate_proj,32.60532906,0.01000,0.599
13,mlp.down_proj,3.95873230,0.01000,3.753
14,self_attn.k_proj,2.99430476,0.01000,0.563
14,self_attn.v_proj,1.20193668,0.01000,0.557
14,self_attn.q_proj,15.81873781,0.01000,0.563
14,self_attn.o_proj,2.06551350,0.01000,0.634
14,mlp.up_proj,33.95725983,0.01000,0.578
14,mlp.gate_proj,33.69845680,0.01000,0.593
14,mlp.down_proj,4.71709626,0.01000,3.694
15,self_attn.k_proj,2.77116690,0.01000,0.609
15,self_attn.v_proj,1.10220412,0.01000,0.591
15,self_attn.q_proj,12.72785654,0.01000,0.599
15,self_attn.o_proj,1.87682963,0.01000,0.609
15,mlp.up_proj,31.97431222,0.01000,0.659
15,mlp.gate_proj,30.82866852,0.01000,0.627
15,mlp.down_proj,4.68322897,0.01000,3.653
16,self_attn.k_proj,2.65178245,0.01000,0.597
16,self_attn.v_proj,1.33414724,0.01000,0.595
16,self_attn.q_proj,13.40043761,0.01000,0.587
16,self_attn.o_proj,2.56995311,0.01000,0.605
16,mlp.up_proj,34.49688110,0.01000,0.593
16,mlp.gate_proj,33.07141986,0.01000,0.594
16,mlp.down_proj,4.67668955,0.01000,3.801
17,self_attn.k_proj,2.56790221,0.01000,0.573
17,self_attn.v_proj,1.63155973,0.01000,0.569
17,self_attn.q_proj,14.37411106,0.01000,0.567
17,self_attn.o_proj,2.11580409,0.01000,0.613
17,mlp.up_proj,40.19091352,0.01000,0.624
17,mlp.gate_proj,37.72127967,0.01000,0.617
17,mlp.down_proj,6.41667661,0.01000,3.904
18,self_attn.k_proj,2.01086315,0.01000,0.627
18,self_attn.v_proj,1.82605520,0.01000,0.622
18,self_attn.q_proj,12.26445795,0.01000,0.628
18,self_attn.o_proj,2.60375627,0.01000,0.626
18,mlp.up_proj,42.72995571,0.01000,0.637
18,mlp.gate_proj,39.51592978,0.01000,0.633
18,mlp.down_proj,7.30472941,0.01000,3.770
19,self_attn.k_proj,2.03734621,0.01000,0.609
19,self_attn.v_proj,2.17895159,0.01000,0.608
19,self_attn.q_proj,13.97130506,0.01000,0.602
19,self_attn.o_proj,2.98630336,0.01000,0.589
19,mlp.up_proj,45.40066239,0.01000,0.608
19,mlp.gate_proj,43.89151394,0.01000,0.612
19,mlp.down_proj,7.82231198,0.01000,3.688
20,self_attn.k_proj,2.03618901,0.01000,0.600
20,self_attn.v_proj,2.32651697,0.01000,0.609
20,self_attn.q_proj,12.49319835,0.01000,0.644
20,self_attn.o_proj,2.02054115,0.01000,0.607
20,mlp.up_proj,53.92699021,0.01000,0.606
20,mlp.gate_proj,51.74172017,0.01000,0.599
20,mlp.down_proj,14.09048887,0.01000,3.809
21,self_attn.k_proj,2.22723000,0.01000,0.632
21,self_attn.v_proj,3.41219732,0.01000,0.620
21,self_attn.q_proj,14.44116497,0.01000,0.618
21,self_attn.o_proj,4.64260670,0.01000,0.616
21,mlp.up_proj,64.68185143,0.01000,0.632
21,mlp.gate_proj,65.52687134,0.01000,0.628
21,mlp.down_proj,17.97917598,0.01000,3.809
22,self_attn.k_proj,2.89653644,0.01000,0.588
22,self_attn.v_proj,5.82066236,0.01000,0.582
22,self_attn.q_proj,18.60508524,0.01000,0.594
22,self_attn.o_proj,5.02134183,0.01000,0.634
22,mlp.up_proj,90.02371180,0.01000,0.611
22,mlp.gate_proj,91.14663930,0.01000,0.615
22,mlp.down_proj,34.05776069,0.01000,3.731
23,self_attn.k_proj,4.18426553,0.01000,0.582
23,self_attn.v_proj,10.33130930,0.01000,0.578
23,self_attn.q_proj,25.64362199,0.01000,0.571
23,self_attn.o_proj,13.37783952,0.01000,0.588
23,mlp.up_proj,125.68233908,0.01000,0.604
23,mlp.gate_proj,128.76723017,0.01000,0.603
23,mlp.down_proj,43.81595416,0.01000,3.752
24,self_attn.k_proj,3.36082683,0.01000,0.582
24,self_attn.v_proj,10.30109492,0.01000,0.590
24,self_attn.q_proj,23.33862463,0.01000,0.593
24,self_attn.o_proj,5.05974270,0.01000,0.585
24,mlp.up_proj,139.16557624,0.01000,0.603
24,mlp.gate_proj,132.34724480,0.01000,0.586
24,mlp.down_proj,59.95090011,0.01000,3.698
25,self_attn.k_proj,4.10498364,0.01000,0.576
25,self_attn.v_proj,17.18356073,0.01000,0.562
25,self_attn.q_proj,27.32380730,0.01000,0.571
25,self_attn.o_proj,9.62589656,0.01000,0.578
25,mlp.up_proj,171.75121912,0.01000,0.598
25,mlp.gate_proj,154.04677341,0.01000,0.583
25,mlp.down_proj,83.74196908,0.01000,3.576
26,self_attn.k_proj,5.83665500,0.01000,0.589
26,self_attn.v_proj,42.81872257,0.01000,0.590
26,self_attn.q_proj,46.47709586,0.01000,0.595
26,self_attn.o_proj,23.30883757,0.01000,0.583
26,mlp.up_proj,170.47679616,0.01000,0.612
26,mlp.gate_proj,150.27404706,0.01000,0.595
26,mlp.down_proj,0.00017362,0.01250,4.201
27,self_attn.k_proj,6.51690253,0.01000,0.592
27,self_attn.v_proj,61.79766743,0.01000,0.568
27,self_attn.q_proj,64.76963792,0.01000,0.587
27,self_attn.o_proj,43.01269404,0.01000,0.577
27,mlp.up_proj,221.63391986,0.01000,0.607
27,mlp.gate_proj,214.62520319,0.01000,0.615
27,mlp.down_proj,415.36760403,0.01000,3.916