Compare text

Find the difference between two text files

Real-time diff

Unified diff

Collapse lines

Highlight change

Syntax highlighting

Tools

Diffchecker Desktop The most secure way to run Diffchecker. Get the Diffchecker Desktop app: your diffs never leave your computer!Get Desktop

dora before vs after

Created 9 months agoDiff never expires

550 lines

547 lines

{

'freq_emb': {

'emb_var': array([[ 0.05383475, -0.02290652, -0.02175547, ..., -0.06938078,

'emb_var': Array([[ 0.05383475, -0.02290652, -0.02175547, ..., -0.06938078,

0.0495423 , 0.00398525],

[ 0.04227516, -0.02249096, 0.05041242, ..., 0.01230512,

-0.03456249, -0.10940999],

[-0.06831896, 0.14380653, -0.04166476, ..., 0.11798981,

0.11934441, 0.00524181]], dtype=float32)

'horizon_ff_layer': {

'hidden_layer': {

'bias': {

'b': array([-0.01867074, -0.00871083, -0.02053265, ..., -0.00441845,

'b': Array([-0.01867074, -0.00871083, -0.02053265, ..., -0.00441845,

-0.01340919, -0.00125625], dtype=float32)

'linear': {

'w': array([[-0.00122285, 0.04531502, -0.01212596, ..., -0.00281182,

'w': Array([[-0.00122285, 0.04531502, -0.01212596, ..., -0.00281182,

0.02374083, 0.02576321],

[-0.0382062 , 0.0059988 , -0.02117914, ..., 0.06201293,

-0.00809431, 0.00410271],

[-0.00531332, 0.00811246, 0.00494867, ..., -0.04757487,

-0.00602691, -0.00266292],

...,

[ 0.03347114, 0.03805783, -0.013433 , ..., 0.01496901,

0.00796273, 0.06580582],

[-0.04087714, -0.00858165, -0.06531095, ..., -0.03156528,

0.0158751 , -0.01732916],

[ 0.05645538, -0.01491658, -0.03915612, ..., 0.01406249,

-0.02100163, 0.01975475]], dtype=float32)

}

'output_layer': {

'bias': {

'b': array([-6.3380185e-03, -9.0496093e-03, -4.2588129e-03, ...,

'b': Array([-6.3380185e-03, -9.0496093e-03, -4.2588129e-03, ...,

-4.3446473e-03, -6.0337975e-06, -3.0872226e-03], dtype=float32)

'linear': {

'w': array([[-0.05310262, -0.02206682, 0.0271988 , ..., -0.01757539,

'w': Array([[-0.05310262, -0.02206682, 0.0271988 , ..., -0.01757539,

-0.02656861, 0.00936702],

[ 0.00877119, -0.0578129 , -0.03252782, ..., -0.0356226 ,

-0.02773722, -0.02957312],

[-0.0370421 , 0.00511809, -0.05719691, ..., 0.00449829,

-0.02505858, -0.02064183],

...,

[ 0.05921374, 0.08082739, 0.08133432, ..., -0.03276784,

-0.05346885, -0.00108078],

[ 0.02845641, 0.03454759, 0.02899572, ..., -0.03797804,

-0.00647498, -0.0080282 ],

[-0.06980696, -0.065199 , -0.09629356, ..., -0.01494522,

-0.00420657, 0.03861954]], dtype=float32)

}

'residual_layer': {

'bias': {

'b': array([-0.00071009, -0.00952863, -0.01032941, ..., 0.00253608,

'b': Array([-0.00071009, -0.00952863, -0.01032941, ..., 0.00253608,

0.00484881, -0.00057579], dtype=float32)

'linear': {

'w': array([[-8.0529033e-05, -8.1122480e-03, -1.1188935e-02, ...,

'w': Array([[-8.0529033e-05, -8.1122480e-03, -1.1188935e-02, ...,

2.7386495e-03, 4.1057365e-03, 3.1871016e-03],

[-2.5267012e-02, -2.9770406e-02, -2.1402119e-02, ...,

9.0102842e-03, 1.3215628e-02, 8.5849436e-03],

[-3.2064769e-02, -9.2049723e-04, -8.8477237e-03, ...,

-2.2026885e-02, -2.9883629e-02, -4.2654805e-02],

...,

[-4.1601930e-02, -1.1100663e-02, -2.1980958e-02, ...,

3.9413955e-02, 4.5028899e-02, 4.5041710e-02],

[ 8.9835078e-03, 1.0722069e-03, -1.1620998e-02, ...,

-3.4306385e-02, -2.8038949e-02, -2.8952893e-02],

[-1.2415100e-02, -2.3502124e-02, -1.4703525e-02, ...,

-2.9797151e-03, 1.5994614e-03, 8.6680241e-04]], dtype=float32)

}

'input_ff_layer': {

'hidden_layer': {

'bias': {

'b': array([-0.01302247, -0.0093135 , -0.00869382, ..., -0.02097026,

'b': Array([-0.01302247, -0.0093135 , -0.00869382, ..., -0.02097026,

-0.00173573, -0.00588994], dtype=float32)

'linear': {

'w': array([[-0.02687993, -0.03276388, 0.0118101 , ..., 0.01630807,

'w': Array([[-0.02687993, -0.03276388, 0.0118101 , ..., 0.01630807,

0.02635014, 0.01112787],

[ 0.03501734, 0.03296082, -0.00576759, ..., 0.01727151,

-0.04001599, -0.00386006],

[ 0.04767988, 0.01772386, 0.04430255, ..., -0.00390798,

-0.02813526, 0.02526983],

...,

[ 0.00559828, -0.02095459, 0.01350581, ..., 0.01727615,

0.03876091, 0.01288361],

[-0.04138084, 0.03345452, -0.00055608, ..., 0.06449523,

0.02750022, -0.01134627],

[-0.04997209, 0.05873165, 0.05058305, ..., -0.01771185,

-0.04247753, -0.01341925]], dtype=float32)

}

'output_layer': {

'bias': {

'b': array([-0.0011562 , 0.00012631, 0.00202118, ..., 0.00015547,

'b': Array([-0.0011562 , 0.00012631, 0.00202118, ..., 0.00015547,

0.00136226, -0.00038068], dtype=float32)

'linear': {

'w': array([[-0.01875325, 0.01847108, -0.03379152, ..., 0.01269644,

'w': Array([[-0.01875325, 0.01847108, -0.03379152, ..., 0.01269644,

0.00307103, -0.00174853],

[ 0.06125693, -0.00906967, -0.03221472, ..., 0.03700627,

-0.03955566, 0.0085879 ],

[-0.01743414, 0.04761524, -0.01841931, ..., -0.03325716,

0.01393927, 0.0339308 ],

...,

[-0.00523872, -0.03900931, 0.01258594, ..., 0.05073977,

0.0148632 , -0.00127181],

[ 0.00715443, 0.01426611, -0.02727203, ..., 0.01418048,

-0.02707596, -0.01978715],

[-0.0379927 , 0.04774408, 0.01647678, ..., -0.0322981 ,

-0.02338383, -0.00932761]], dtype=float32)

}

'residual_layer': {

'bias': {

'b': array([-0.00342117, 0.0011725 , -0.00115519, ..., -0.01340925,

'b': Array([-0.00342117, 0.0011725 , -0.00115519, ..., -0.01340925,

-0.01195502, -0.0078334 ], dtype=float32)

'linear': {

'w': array([[ 0.060296 , 0.05077375, 0.06998359, ..., -0.03257499,

'w': Array([[ 0.060296 , 0.05077375, 0.06998359, ..., -0.03257499,

0.05363302, -0.049019 ],

[ 0.03054528, -0.03217815, -0.0089082 , ..., -0.04977859,

-0.02510578, 0.01701915],

[-0.04686454, 0.03704751, -0.02275803, ..., 0.0496948 ,

0.03227101, 0.0320745 ],

...,

[ 0.07358721, 0.05074072, 0.06101955, ..., 0.0217406 ,

-0.00013694, 0.01635611],

[ 0.04090381, 0.07156919, 0.00553412, ..., -0.04919641,

0.05928294, -0.05756657],

[-0.03001764, -0.01901046, 0.0523936 , ..., -0.05530353,

0.04039487, -0.01157674]], dtype=float32)

}

'stacked_transformer_layer': {

'x_layers_0': {

'ff_layer': {

'ffn_layer1': {

'bias': {

'b': array([ 0.00422558, 0.00153717, -0.00328633, ..., 0.01313652,

'b': Array([ 0.00422558, 0.00153717, -0.00328633, ..., 0.01313652,

0.01618191, 0.0046046 ], dtype=float32)

'linear': {

'w': array([[-0.04934547, 0.03615815, 0.02133694, ..., 0.00294278,

'dora_m': Array([[1.0408473 , 0.99524635, 0.98682207, ..., 1.1109077 , 1.0122046 ,

1.0305351 ]], dtype=float32),

'lora_a': Array([[-0.10847064],

[ 0.294551 ],

[-0.26035047],

...,

[-0.79824907],

[ 0.73761564],

[-0.08403018]], dtype=float32),

'lora_b': Array([[ 5.1776869e-06],

[ 1.5813343e-05],

[-1.4205739e-05],

...,

[-9.6916729e-06],

[-1.8683011e-06],

[-5.3805843e-06]], dtype=float32),

'w': Array([[-0.04934547, 0.03615815, 0.02133694, ..., 0.00294278,

0.03395786, -0.02778906],

[-0.00490594, 0.03894801, -0.02373052, ..., -0.03631651,

0.0288607 , 0.05464533],

[-0.00822064, -0.0459388 , 0.02931811, ..., 0.0401447 ,

-0.0274516 , -0.0161763 ],

...,

[-0.01827895, 0.05226748, 0.01714141, ..., -0.005661 ,

0.02436945, 0.01169135],

[-0.03512175, 0.0176127 , 0.02929639, ..., 0.0022374 ,

0.01826224, 0.03047828],

[-0.00179529, -0.01935859, 0.02216526, ..., -0.01494863,

0.04285045, -0.02584894]], dtype=float32),

0.04285045, -0.02584894]], dtype=float32)

'lora_a': Array([[-0.10846964],

[ 0.29453316],

[-0.2603706 ],

...,

[-0.7982575 ],

[ 0.7376351 ],

[-0.08402041]], dtype=float32),

'lora_b': Array([[0.],

[0.],

...,

[0.],

[0.]], dtype=float32),

'dora_m': Array([[1.0408502, 0.9952368, 0.9868272, ..., 1.110908 , 1.0122148,

1.0305483]], dtype=float32)

}

'ffn_layer2': {

'bias': {

'b': array([ 8.5014108e-05, -3.6904723e-03, -2.2362696e-03, ...,

'b': Array([ 8.5014108e-05, -3.6904723e-03, -2.2362696e-03, ...,

-2.6434378e-03, -4.6255854e-03, -2.7873858e-03], dtype=float32)

'linear': {

'w': array([[-0.00317632, 0.01268681, -0.02180147, ..., -0.03509863,

'dora_m': Array([[1.0201988, 1.0224365, 1.0201229, ..., 1.0271175, 1.0138276,

1.0606217]], dtype=float32),

'lora_a': Array([[-0.10848837],

[ 0.29451126],

[-0.26038843],

...,

[-0.7982782 ],

[ 0.737633 ],

[-0.08402973]], dtype=float32),

'lora_b': Array([[ 9.4581455e-06],

[-5.8203859e-06],

[-4.4713113e-07],

...,

[ 1.9027753e-05],

[-4.2390179e-06],

[-1.8873763e-05]], dtype=float32),

'w': Array([[-0.00317632, 0.01268681, -0.02180147, ..., -0.03509863,

0.02900657, 0.05286129],

[ 0.03748895, -0.01666473, 0.04159482, ..., 0.01672644,

-0.03867533, 0.03111953],

[-0.04813586, -0.03120456, 0.02479169, ..., 0.01334335,

-0.04884216, 0.00229996],

...,

[-0.03854724, 0.0137933 , -0.01018869, ..., -0.0093796 ,

0.03359092, 0.00674478],

[ 0.0059021 , -0.00476607, -0.01634962, ..., 0.02304816,

-0.01485625, 0.01752446],

[-0.02087662, 0.00162849, 0.0099754 , ..., -0.06226157,

0.02198281, -0.00740848]], dtype=float32),

0.02198281, -0.00740848]], dtype=float32)

'lora_a': Array([[-0.10846964],

[ 0.29453316],

[-0.2603706 ],

...,

[-0.7982575 ],

[ 0.7376351 ],

[-0.08402041]], dtype=float32),

'lora_b': Array([[0.],

[0.],

...,

[0.],

[0.]], dtype=float32),

'dora_m': Array([[1.0201908, 1.0224168, 1.0201058, ..., 1.0270991, 1.013838 ,

1.0606363]], dtype=float32)

}

'layer_norm': {

'bias': array([ 0.00631881, 0.01150341, 0.00914302, ..., -0.0200112 ,

'bias': Array([ 0.00631881, 0.01150341, 0.00914302, ..., -0.0200112 ,

-0.00163537, -0.000495 ], dtype=float32),

'scale': array([-0.03739248, -0.0243671 , -0.02632998, ..., -0.01486455,

'scale': Array([-0.03739248, -0.0243671 , -0.02632998, ..., -0.01486455,

0.01445367, 0.04890316], dtype=float32)

}

'layer_norm': {

'scale': array([1.2228152 , 1.1309394 , 1.2290405 , ..., 0.96604544, 0.9487428 ,

'scale': Array([1.2228152 , 1.1309394 , 1.2290405 , ..., 0.96604544, 0.9487428 ,

0.9532705 ], dtype=float32)

'self_attention': {

'key': {

'b': array([[-6.4075930e-06, 4.2504727e-04, -1.1618993e-03, ...,

'b': Array([[-6.4075930e-06, 4.2504727e-04, -1.1618993e-03, ...,

-1.7168164e-03, 2.2836959e-04, -8.8550907e-05],

[ 3.1805394e-04, 3.4957723e-04, 5.1292509e-04, ...,

2.6313388e-03, 1.0807015e-03, -2.1224995e-03],

[ 9.5978525e-04, -3.6766953e-03, -1.1368678e-03, ...,

1.4873255e-03, -2.3283053e-03, -4.0190420e-03],

...,

[-1.6945685e-03, -1.5151855e-03, -1.4960926e-03, ...,

1.8141167e-03, -2.3590028e-03, -2.6155929e-03],

[-5.4188799e-03, 4.6130762e-04, -6.1691484e-05, ...,

3.2484343e-03, -2.7115741e-03, -3.6162001e-04],

[-2.8124801e-03, 1.5307369e-04, -2.2465425e-05, ...,

9.6123409e-04, -2.0123613e-03, -4.0718642e-05]], dtype=float32),

'w': array([[[-2.00603511e-02, 5.75137278e-03, -8.72080214e-03, ...,

'dora_m': Array([[[0.29071984, 0.29683936, 0.30372328, ..., 0.34602427,

0.29602143, 0.305746 ],

[0.32390478, 0.33643064, 0.34294572, ..., 0.43285245,

0.3494392 , 0.33882546],

[0.37621891, 0.43592635, 0.43361625, ..., 0.48496065,

0.41900933, 0.39550942],

...,

[0.6301863 , 0.8522955 , 0.5222311 , ..., 0.4287274 ,

0.5746888 , 0.44996235],

[0.53644997, 0.38934904, 0.48470625, ..., 0.5750841 ,

0.32962137, 0.76050806],

[0.3797709 , 0.32443294, 0.3033168 , ..., 0.2547274 ,

0.25814548, 0.25158834]]], dtype=float32),

'lora_a': Array([[-0.10848147],

[ 0.2945259 ],

[-0.26038498],

...,

[-0.79826516],

[ 0.73763 ],

[-0.08401564]], dtype=float32),

'lora_b': Array([[ 5.0566196e-06],

[-1.0148730e-05],

[ 1.1051195e-05],

...,

[ 6.9015719e-06],

[ 7.3781512e-06],

[-1.4809251e-05]], dtype=float32),

'w': Array([[[-2.00603511e-02, 5.75137278e-03, -8.72080214e-03, ...,

-2.70311963e-02, 1.24247791e-03, -2.36301706e-03],

[ 3.99277313e-03, 2.94813123e-02, -1.99544337e-02, ...,

1.95915089e-03, 1.23697640e-02, 5.81120653e-03],

[-1.01987063e-03, 3.53949121e-03, -6.74259616e-03, ...,

3.45695938e-04, -3.17200739e-03, 1.25762047e-02],

...,

[ 1.12090185e-02, 5.59417754e-02, 2.80567389e-02, ...,

5.72427809e-02, -2.45896727e-02, -4.41132709e-02],

[-2.39311568e-02, 1.08251497e-02, 1.98488291e-02, ...,

2.62170825e-02, -1.07316952e-02, 7.13963956e-02],

[-5.38694439e-03, 2.10208986e-02, 4.16014344e-03, ...,

1.06410738e-02, 1.50726573e-03, -1.23633593e-02]],

[[ 3.49556189e-03, 3.44262924e-04, 2.85613537e-03, ...,

-1.43942228e-02, -5.75408980e-04, 3.76027485e-04],

[ 8.77154060e-03, 1.46433450e-02, -5.25063137e-03, ...,

-3.58833029e-04, 2.95653217e-03, -3.85825592e-03],

[ 2.57922034e-03, -1.56476088e-02, -1.68384165e-02, ...,

1.31746242e-02, -7.18129473e-03, -8.73307057e-04],

...,

[ 2.63401377e-03, 4.11511064e-02, 1.82539616e-02, ...,

3.93974334e-02, -2.09947210e-02, -4.63477969e-02],

[-3.26764248e-02, 4.17060219e-03, 1.78119279e-02, ...,

3.88964377e-02, -1.26539385e-02, 5.79634905e-02],

[ 3.92154464e-03, 1.28147583e-02, -5.92348725e-03, ...,

1.19894780e-02, 4.40148357e-03, -1.42792449e-03]],

[[-5.62313665e-03, -2.09410232e-03, -9.82910208e-03, ...,

-3.35871242e-02, -2.54542637e-03, -3.40952398e-03],

[ 5.54407481e-03, 1.78536046e-02, -7.02433800e-03, ...,

-8.23897962e-03, 6.65213913e-03, -1.49944180e-03],

[ 1.13043087e-02, -1.39708715e-02, -1.11245923e-02, ...,

1.64344721e-02, -8.75243638e-03, 1.43328554e-03],

...,

[-1.51386904e-03, 5.43856286e-02, 5.52712865e-02, ...,

5.97689189e-02, -8.20651092e-03, -4.67697680e-02],

[-2.99225654e-02, -6.27992325e-04, 1.96734313e-02, ...,

5.00066131e-02, -2.26834826e-02, 6.30012304e-02],

[ 7.33884191e-03, 7.73959141e-03, 2.24917941e-03, ...,

1.09161446e-02, -6.46742759e-03, -3.35140945e-03]],

...,

[[ 1.10051483e-02, -1.85540915e-02, 9.35962051e-03, ...,

6.85161399e-03, 5.54813165e-03, -6.58737845e-04],

[-7.81041523e-03, 3.98438890e-03, -1.66073244e-03, ...,

-1.19862985e-03, -1.22908978e-02, -6.26032334e-03],

[ 7.88267050e-03, -2.79578473e-02, -1.36419181e-02, ...,

2.40753926e-02, -1.45410048e-02, -1.76570639e-02],

...,

[-6.93083834e-03, 1.26243681e-02, -1.05324322e-02, ...,

-1.96885108e-03, 1.16551239e-02, -1.18715111e-02],

[-6.10861601e-03, 3.95314310e-05, 1.73927583e-02, ...,

-8.47204216e-03, 1.13081449e-04, 2.14985292e-03],

[-2.08475161e-03, -1.37468986e-02, 5.63349109e-03, ...,

-1.12989545e-02, -8.50978866e-03, 1.94774184e-03]],

[[ 2.74757389e-04, -4.94607608e-04, 6.77777128e-03, ...,

-1.00600952e-02, 8.12825561e-03, -5.75583940e-03],

[ 9.11341235e-03, -1.28179893e-03, -5.31734154e-03, ...,

1.71367656e-02, -1.20344115e-02, -4.43551084e-03],

[ 1.83838531e-02, -1.47979856e-02, -1.40168304e-02, ...,

1.32685406e-02, -2.14968957e-02, -1.05311433e-02],

...,

[-1.04027316e-02, 8.37995019e-03, -1.30986366e-02, ...,

4.06161277e-03, 8.50717258e-03, -1.61217351e-03],

[ 4.98227309e-03, -1.39553854e-02, 4.06156154e-03, ...,

-7.87476171e-03, 5.47154248e-03, -6.78235898e-03],

[-1.80154722e-02, -3.77116865e-03, 8.45899712e-03, ...,

-1.32837438e-03, -1.42556825e-03, -1.07409863e-03]],

[[-1.08604589e-02, 1.28442254e-02, 5.53134922e-03, ...,

-5.09914523e-03, -1.28980596e-02, 1.25545133e-02],

[-1.18277026e-02, 8.53359420e-03, 3.05717532e-03, ...,

1.41720111e-02, -1.60324126e-02, 1.07069116e-03],

[-1.69687124e-03, 8.45217053e-03, 6.06904272e-03, ...,

-1.82976536e-02, 7.91715831e-03, 2.23788200e-03],

...,

[-6.64934842e-03, 6.04555802e-03, -1.44436248e-02, ...,

-7.54461996e-03, 1.34342900e-02, 1.26932003e-03],

[ 8.17108434e-03, 2.05422216e-03, 1.13008972e-02, ...,

-1.15924431e-02, 7.23493379e-03, -1.01354849e-02],

[-3.08707799e-03, -2.84547848e-03, 5.24008553e-03, ...,

-9.74707212e-03, -9.47374292e-03, 8.77883460e-04]]],

-9.74707212e-03, -9.47374292e-03, 8.77883460e-04]]], dtype=float32)

dtype=float32),

'lora_a': Array([[-0.10846964],

[ 0.29453316],

[-0.2603706 ],

...,

[-0.7982575 ],

[ 0.7376351 ],

[-0.08402041]], dtype=float32),

'lora_b': Array([[0.],

[0.],

...,

[0.],

[0.]], dtype=float32),

'dora_m': Array([[[0.29073346, 0.29685742, 0.3037179 , ..., 0.34602788,

0.29602417, 0.30575013],

[0.32391438, 0.33644444, 0.3429411 , ..., 0.43286625,

0.34944308, 0.33880782],

[0.37621805, 0.43590567, 0.43363446, ..., 0.4849415 ,

0.4190295 , 0.3955025 ],

...,

[0.63017344, 0.85228944, 0.5222197 , ..., 0.42874113,

0.574699 , 0.44997627],

[0.53644234, 0.38935047, 0.48470864, ..., 0.5750971 ,

0.32961357, 0.7605141 ],

[0.37976965, 0.32443202, 0.3033245 , ..., 0.25471726,

0.25815964, 0.25158453]]], dtype=float32)

'per_dim_scale': {

'per_dim_scale': array([0.11543996, 0.19667041, 0.08449101, 0.15537748, 0.05297864,

'per_dim_scale': Array([0.11543996, 0.19667041, 0.08449101, 0.15537748, 0.05297864,

0.08272862, 0.08896066, 0.09299482, 0.14027727, 0.12275486,

0.13754928, 0.11227061, 0.07794799, 0.11385844, 0.21155985,

0.07013194, 0.11293491, 0.11814845, 0.14049305, 0.1071273 ,

0.07294399, 0.11549649, 0.10932729, 0.03889112, 0.07322115,

0.09397113, 0.08638889, 0.086013 , 0.09734396, 0.04915351,

0.07681435, 0.0727135 , 0.06417551, 0.1102055 , 0.05094118,

0.06759506, 0.10526094, 0.06561205, 0.08017935, 0.08598939,

0.08295949, 0.10986121, 0.06075855, 0.06381149, 0.09573651,

0.15352455, 0.10654029, 0.05720843, 0.1080655 , 0.08904675,

0.11533722, 0.10753154, 0.14174831, 0.05750328, 0.10738507,

0.13763388, 0.11135354, 0.07917753, 0.05568942, 0.18191479,

0.19961604, 0.08848751, 0.1105781 , 0.11773358, 0.13192818,

0.05453964, 0.07397411, 0.04283143, 0.09388805, 0.12555331,

0.15708339, 0.14256129, 0.08874376, 0.08300523, 0.08548183,

0.07495448, 0.01806459, 0.10216796, 0.08380181, 0.06248857],

0.07495448, 0.01806459, 0.10216796, 0.08380181, 0.06248857], dtype=float32)

dtype=float32)

'post': {

'b': array([ 0.0038859 , 0.00227933, 0.00126942, ..., -0.00807348,

'b': Array([ 0.0038859 , 0.00227933, 0.00126942, ..., -0.00807348,

-0.0033168 , -0.00190115], dtype=float32),

'w': array([[[ 0.01001332, 0.00383801, 0.00365361, ..., -0.00576187,

'dora_m': Array([[[0.19123024, 0.14076011, 0.14161648, ..., 0.15473266,

0.1719282 , 0.14289714],

[0.22512862, 0.25811967, 0.23838049, ..., 0.25743642,

0.24412851, 0.26848155],

[0.15662938, 0.15710527, 0.16476132, ..., 0.13925603,

0.13652036, 0.15013184],

...,

[0.31857708, 0.30544275, 0.28747907, ..., 0.2713666 ,

0.29232302, 0.2772732 ],

[0.29738736, 0.26313943, 0.25538403, ..., 0.27492785,

0.28833735, 0.2847642 ],

[0.2633174 , 0.26364738, 0.2317865 , ..., 0.2222713 ,

0.26653096, 0.24285616]]], dtype=float32),

'lora_a': Array([[-0.10847466],

[ 0.29454225],

[-0.2603705 ],

...,

[-0.7982707 ],

[ 0.7376422 ],

[-0.08400187]], dtype=float32),

'lora_b': Array([[ 1.2972054e-05],

[-3.7403761e-06],

[ 6.7211336e-06],

...,

[ 1.2964535e-05],

[ 1.6402071e-05],

[ 1.2927146e-05]], dtype=float32),

'w': Array([[[ 0.01001332, 0.00383801, 0.00365361, ..., -0.00576187,

0.00987968, 0.00056312],

[ 0.00502719, 0.0106297 , -0.00514524, ..., 0.00606945,

0.00654817, -0.00433787],

[ 0.00358599, -0.00257 , 0.00474215, ..., -0.00819106,

0.0009758 , -0.00469567],

...,

[ 0.00208981, 0.01763259, -0.01958698, ..., 0.01154696,

-0.009217 , 0.00837894],

[-0.0042141 , -0.00337535, -0.01742578, ..., -0.00651149,

0.008242 , 0.00044077],

[-0.00331672, 0.00542616, 0.00861929, ..., 0.00442049,

-0.00129945, 0.00199285]],

[[-0.00784615, 0.0007228 , -0.00331567, ..., -0.00073429,

0.00443421, 0.00215626],

[ 0.00290094, 0.00959504, -0.00379133, ..., -0.00251239,

0.00683427, -0.00107421],

[-0.00282467, 0.00305181, -0.0018921 , ..., -0.0041155 ,

0.00290548, -0.00524714],

...,

[-0.0083752 , 0.00948612, 0.01147135, ..., 0.00237548,

-0.00356866, 0.00163063],

[-0.00435623, -0.02190368, -0.02925595, ..., -0.01722581,

-0.01212152, 0.00048348],

[-0.00665992, -0.00317996, 0.00328454, ..., 0.00392423,

0.00951003, -0.00905988]],

[[-0.00525643, 0.0023299 , -0.00355807, ..., 0.00550958,

0.00718864, -0.00195274],

[ 0.00557126, 0.0047749 , 0.0013244 , ..., 0.0020559 ,

0.00676667, 0.0004616 ],

[ 0.0036528 , 0.00101903, -0.00275526, ..., -0.00442425,

-0.00125036, -0.00322119],

...,

[ 0.01212482, 0.00698769, -0.00469012, ..., 0.0119752 ,

-0.01411841, 0.02146803],

[ 0.00513848, -0.00179516, -0.0260424 , ..., -0.01782374,

-0.00823828, 0.00242193],

[-0.00726525, -0.00548198, 0.00631244, ..., 0.00095198,

0.00166597, -0.01111033]],

...,

[[ 0.00183661, 0.00537889, 0.00282976, ..., 0.00681035,

-0.00287456, 0.00020768],

[-0.00195633, -0.01244443, -0.00306213, ..., -0.00138315,

0.00965622, -0.00352222],

[ 0.00638766, -0.00434901, -0.00251938, ..., 0.0082688 ,

0.00016049, 0.00494107],

...,

[-0.00437414, -0.00500601, 0.00331668, ..., -0.01888177,

0.00244803, -0.00266423],

[-0.01324576, 0.00685205, -0.00709169, ..., 0.00598535,

-0.00090406, -0.00340358],

[ 0.00550575, 0.00696265, -0.00663389, ..., 0.01093742,

-0.0047209 , -0.00974895]],

[[-0.00350233, -0.00138139, -0.00094495, ..., -0.00453996,

-0.00188128, 0.00299819],

[ 0.00179204, 0.00947329, -0.00054177, ..., 0.00415827,

-0.0054397 , 0.00070148],

[ 0.0055178 , -0.00677183, -0.00414257, ..., -0.00434643,

0.00546411, -0.00096053],

...,

[ 0.00150547, -0.01467462, 0.00350618, ..., -0.00134948,

0.01102407, 0.00321508],

[ 0.0170079 , 0.00656268, -0.00057632, ..., 0.00752047,

-0.00078745, 0.00861916],

[ 0.00960819, -0.00827104, -0.00167912, ..., 0.00267513,

-0.00212967, 0.00015723]],

[[ 0.00066472, -0.00265332, -0.00420681, ..., 0.00356282,

-0.00272015, -0.00122229],

[-0.00913083, -0.00096118, -0.00214067, ..., -0.00021383,

-0.00040713, -0.0020092 ],

[-0.0020909 , 0.00057581, -0.00198702, ..., -0.00334667,

-0.00107645, -0.00010455],

...,

[-0.0086949 , -0.00821191, 0.00208952, ..., -0.00145943,

-0.01484594, -0.00135062],

[ 0.00198201, 0.000815 , 0.00522504, ..., 0.00207165,

0.0071472 , -0.00770016],

[ 0.01279963, 0.00408498, -0.00312828, ..., 0.00053384,

-0.00386579, -0.00476406]]], dtype=float32),

-0.00386579, -0.00476406]]], dtype=float32)

'lora_a': Array([[-0.10846964],

[ 0.29453316],

[-0.2603706 ],

...,

[-0.7982575 ],

[ 0.7376351 ],

[-0.08402041]], dtype=float32),

'lora_b': Array([[0.],

[0.],

...,

[0.],

[0.]], dtype=float32),

'dora_m': Array([[[0.19123395, 0.14074375, 0.1416127 , ..., 0.1547206 ,

0.1719081 , 0.14288525],

[0.22511503, 0.2581373 , 0.23838745, ..., 0.2574504 ,

0.24413465, 0.26848358],

[0.15661407, 0.1571259 , 0.16474354, ..., 0.13923596,

0.13650616, 0.15014726],

...,

[0.31857383, 0.30543646, 0.28746378, ..., 0.27137437,

0.2923366 , 0.27726355],

[0.2973705 , 0.2631207 , 0.25538802, ..., 0.27492747,

0.28831685, 0.28475752],

[0.26332352, 0.26362932, 0.2317942 , ..., 0.22226565,

0.2665483 , 0.24287087]]], dtype=float32)

'query': {

'b': array([[-0.05727649, 0.04552313, -0.04670092, ..., 0.00146665,

'b': Array([[-0.05727649, 0.04552313, -0.04670092, ..., 0.00146665,

0.02421395, -0.00091885],

[-0.0103084 , 0.00550128, -0.01701137, ..., 0.02177138,

0.01786684, 0.00672262],

[-0.19440088, 0.23860118, 0.19767429, ..., -0.2909671 ,

0.26273695, 0.06778763],

...,

[-0.01284689, -0.00560991, -0.00838155, ..., -0.00923357,

0.00546553, 0.01274918],

[-0.04595913, 0.00850777, -0.00376768, ..., 0.00486895,

-0.06195865, -0.02272983],

[-0.07012997, -0.08104558, 0.06873379, ..., 0.04658027,

-0.02241261, 0.02544953]], dtype=float32),

'w': array([[[-1.8535994e-02, 9.5346302e-04, -1.9991446e-02, ...,

'dora_m': Array([[[0.46142048, 0.4500186 , 0.34651643, ..., 0.31194046,

0.3929578 , 0.32459253],

[0.2763315 , 0.2899008 , 0.4391211 , ..., 0.49179116,

0.39203978, 0.41947088],

[0.3319017 , 0.80007493, 0.48443404, ..., 0.65731555,

0.45607182, 0.6975951 ],

...,

[0.6744632 , 0.6936384 , 0.5252137 , ..., 0.4535748 ,

0.5806035 , 0.4434118 ],

[0.71431696, 0.38224635, 0.44784164, ..., 0.5832321 ,

0.3745505 , 0.62699455],

[0.32467127, 0.36532265, 0.2966676 , ..., 0.35534376,

0.2847552 , 0.33106232]]], dtype=float32),

'lora_a': Array([[-0.10846767],

[ 0.29452536],

[-0.26035994],

...,

[-0.79827595],

[ 0.7376163 ],

[-0.08403978]], dtype=float32),

'lora_b': Array([[-2.1385622e-05],

[ 2.1661906e-05],

[ 3.5902622e-06],

...,

[ 1.3643391e-05],

[-3.5790296e-07],

[-2.8833008e-06]], dtype=float32),

'w': Array([[[-1.8535994e-02, 9.5346302e-04, -1.9991446e-02, ...,

1.0161528e-02, -1.9445473e-02, 2.0938339e-02],

[ 4.7027450e-02, -1.9295773e-02, 5.2923034e-03, ...,

-1.1656388e-02, 6.4211138e-02, 8.2570147e-03],

[-7.1002268e-03, 1.9432401e-02, 3.4478415e-02, ...,

-2.4406184e-02, 8.6701987e-03, 2.6081081e-02],

...,

[ 8.1564359e-02, 9.4508991e-02, 9.0464605e-03, ...,

1.5598202e-02, -5.1820800e-02, -2.5631530e-02],

[ 8.2158990e-02, -1.8991157e-02, -4.1691676e-02, ...,

-4.6170041e-02, 3.6681116e-02, -4.2611934e-02],

[-8.7982453e-03, -5.2305171e-03, 2.0775546e-03, ...,

-3.1847455e-02, -2.3901056e-02, 2.6486890e-02]],

[[-1.8576467e-02, 8.5971802e-03, -1.9604284e-02, ...,

4.1900217e-03, 7.2023519e-03, 1.5307357e-02],

[ 8.4553892e-03, -1.5333053e-02, 1.1094856e-02, ...,

[ 8.4553892e-03, -1.5333053e-02, 1.1094856e-02, .

-2.2008531e-03, 3.6183935e-02, -8.0536548e-03],

[ 6.3647947e-04, 2.3599932e-02, 2.9660890e-02, ...,

-1.9744297e-02, 2.0901676e-02, 1.1683471e-02],

...,

[ 6.4568944e-02, 8.3643161e-02, 1.1189053e-02, ...,

2.0544671e-02, -5.1902432e-02, -1.3179816e-02],

[ 8.4815398e-02, -5.9528677e-03, -3.0726269e-02, ...,

-5.4053325e-02, 2.0459246e-02, -2.3801735e-02],

[-6.9145137e-03, -1.0658662e-02, 2.6543209e-02, ...,

-1.8605150e-02, -2.5469976e-02, 3.2749582e-02]],

[[-6.6796737e-03, 4.0878244e-03, -1.5109884e-02, ...,

-1.8557297e-02, -8.3989222e-03, 1.9515160e-02],

[ 2.6753681e-02, 5.5092634e-03, -9.7639320e-05, ...,

-1.6482458e-02, 4.7398698e-02, -7.3832017e-04],

[-5.1337848e-03, 6.3418280e-03, 2.3368003e-02, ...,

-1.6323408e-02, 2.7475895e-03, 1.3939449e-02],

...,

[ 5.7961229e-02, 1.1082336e-01, 1.8057462e-02, ...,

2.1232987e-02, -4.0269952e-02, -9.0204747e-03],

[ 9.7470656e-02, 6.0731932e-03, -2.4925102e-02, ...,

-7.1226001e-02, 4.1544825e-02, -3.4857236e-02],

[ 1.8736666e-02, -1.1518041e-03, 1.4533965e-02, ...,

-4.3398786e-02, -2.3378823e-02, 4.3798864e-02]],

...,

[[-9.1594533e-04,

Saved diffs

Original text

Open file

{
    'freq_emb': {
        'emb_var': array([[ 0.05383475, -0.02290652, -0.02175547, ..., -0.06938078,
         0.0495423 ,  0.00398525],
       [ 0.04227516, -0.02249096,  0.05041242, ...,  0.01230512,
        -0.03456249, -0.10940999],
       [-0.06831896,  0.14380653, -0.04166476, ...,  0.11798981,
         0.11934441,  0.00524181]], dtype=float32)
    },
    'horizon_ff_layer': {
        'hidden_layer': {
            'bias': {
                'b': array([-0.01867074, -0.00871083, -0.02053265, ..., -0.00441845,
       -0.01340919, -0.00125625], dtype=float32)
            },
            'linear': {
                'w': array([[-0.00122285,  0.04531502, -0.01212596, ..., -0.00281182,
         0.02374083,  0.02576321],
       [-0.0382062 ,  0.0059988 , -0.02117914, ...,  0.06201293,
        -0.00809431,  0.00410271],
       [-0.00531332,  0.00811246,  0.00494867, ..., -0.04757487,
        -0.00602691, -0.00266292],
       ...,
       [ 0.03347114,  0.03805783, -0.013433  , ...,  0.01496901,
         0.00796273,  0.06580582],
       [-0.04087714, -0.00858165, -0.06531095, ..., -0.03156528,
         0.0158751 , -0.01732916],
       [ 0.05645538, -0.01491658, -0.03915612, ...,  0.01406249,
        -0.02100163,  0.01975475]], dtype=float32)
            }
        },
        'output_layer': {
            'bias': {
                'b': array([-6.3380185e-03, -9.0496093e-03, -4.2588129e-03, ...,
       -4.3446473e-03, -6.0337975e-06, -3.0872226e-03], dtype=float32)
            },
            'linear': {
                'w': array([[-0.05310262, -0.02206682,  0.0271988 , ..., -0.01757539,
        -0.02656861,  0.00936702],
       [ 0.00877119, -0.0578129 , -0.03252782, ..., -0.0356226 ,
        -0.02773722, -0.02957312],
       [-0.0370421 ,  0.00511809, -0.05719691, ...,  0.00449829,
        -0.02505858, -0.02064183],
       ...,
       [ 0.05921374,  0.08082739,  0.08133432, ..., -0.03276784,
        -0.05346885, -0.00108078],
       [ 0.02845641,  0.03454759,  0.02899572, ..., -0.03797804,
        -0.00647498, -0.0080282 ],
       [-0.06980696, -0.065199  , -0.09629356, ..., -0.01494522,
        -0.00420657,  0.03861954]], dtype=float32)
            }
        },
        'residual_layer': {
            'bias': {
                'b': array([-0.00071009, -0.00952863, -0.01032941, ...,  0.00253608,
        0.00484881, -0.00057579], dtype=float32)
            },
            'linear': {
                'w': array([[-8.0529033e-05, -8.1122480e-03, -1.1188935e-02, ...,
         2.7386495e-03,  4.1057365e-03,  3.1871016e-03],
       [-2.5267012e-02, -2.9770406e-02, -2.1402119e-02, ...,
         9.0102842e-03,  1.3215628e-02,  8.5849436e-03],
       [-3.2064769e-02, -9.2049723e-04, -8.8477237e-03, ...,
        -2.2026885e-02, -2.9883629e-02, -4.2654805e-02],
       ...,
       [-4.1601930e-02, -1.1100663e-02, -2.1980958e-02, ...,
         3.9413955e-02,  4.5028899e-02,  4.5041710e-02],
       [ 8.9835078e-03,  1.0722069e-03, -1.1620998e-02, ...,
        -3.4306385e-02, -2.8038949e-02, -2.8952893e-02],
       [-1.2415100e-02, -2.3502124e-02, -1.4703525e-02, ...,
        -2.9797151e-03,  1.5994614e-03,  8.6680241e-04]], dtype=float32)
            }
        }
    },
    'input_ff_layer': {
        'hidden_layer': {
            'bias': {
                'b': array([-0.01302247, -0.0093135 , -0.00869382, ..., -0.02097026,
       -0.00173573, -0.00588994], dtype=float32)
            },
            'linear': {
                'w': array([[-0.02687993, -0.03276388,  0.0118101 , ...,  0.01630807,
         0.02635014,  0.01112787],
       [ 0.03501734,  0.03296082, -0.00576759, ...,  0.01727151,
        -0.04001599, -0.00386006],
       [ 0.04767988,  0.01772386,  0.04430255, ..., -0.00390798,
        -0.02813526,  0.02526983],
       ...,
       [ 0.00559828, -0.02095459,  0.01350581, ...,  0.01727615,
         0.03876091,  0.01288361],
       [-0.04138084,  0.03345452, -0.00055608, ...,  0.06449523,
         0.02750022, -0.01134627],
       [-0.04997209,  0.05873165,  0.05058305, ..., -0.01771185,
        -0.04247753, -0.01341925]], dtype=float32)
            }
        },
        'output_layer': {
            'bias': {
                'b': array([-0.0011562 ,  0.00012631,  0.00202118, ...,  0.00015547,
        0.00136226, -0.00038068], dtype=float32)
            },
            'linear': {
                'w': array([[-0.01875325,  0.01847108, -0.03379152, ...,  0.01269644,
         0.00307103, -0.00174853],
       [ 0.06125693, -0.00906967, -0.03221472, ...,  0.03700627,
        -0.03955566,  0.0085879 ],
       [-0.01743414,  0.04761524, -0.01841931, ..., -0.03325716,
         0.01393927,  0.0339308 ],
       ...,
       [-0.00523872, -0.03900931,  0.01258594, ...,  0.05073977,
         0.0148632 , -0.00127181],
       [ 0.00715443,  0.01426611, -0.02727203, ...,  0.01418048,
        -0.02707596, -0.01978715],
       [-0.0379927 ,  0.04774408,  0.01647678, ..., -0.0322981 ,
        -0.02338383, -0.00932761]], dtype=float32)
            }
        },
        'residual_layer': {
            'bias': {
                'b': array([-0.00342117,  0.0011725 , -0.00115519, ..., -0.01340925,
       -0.01195502, -0.0078334 ], dtype=float32)
            },
            'linear': {
                'w': array([[ 0.060296  ,  0.05077375,  0.06998359, ..., -0.03257499,
         0.05363302, -0.049019  ],
       [ 0.03054528, -0.03217815, -0.0089082 , ..., -0.04977859,
        -0.02510578,  0.01701915],
       [-0.04686454,  0.03704751, -0.02275803, ...,  0.0496948 ,
         0.03227101,  0.0320745 ],
       ...,
       [ 0.07358721,  0.05074072,  0.06101955, ...,  0.0217406 ,
        -0.00013694,  0.01635611],
       [ 0.04090381,  0.07156919,  0.00553412, ..., -0.04919641,
         0.05928294, -0.05756657],
       [-0.03001764, -0.01901046,  0.0523936 , ..., -0.05530353,
         0.04039487, -0.01157674]], dtype=float32)
            }
        }
    },
    'stacked_transformer_layer': {
        'x_layers_0': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': array([ 0.00422558,  0.00153717, -0.00328633, ...,  0.01313652,
        0.01618191,  0.0046046 ], dtype=float32)
                    },
                    'linear': {
                        'w': array([[-0.04934547,  0.03615815,  0.02133694, ...,  0.00294278,
         0.03395786, -0.02778906],
       [-0.00490594,  0.03894801, -0.02373052, ..., -0.03631651,
         0.0288607 ,  0.05464533],
       [-0.00822064, -0.0459388 ,  0.02931811, ...,  0.0401447 ,
        -0.0274516 , -0.0161763 ],
       ...,
       [-0.01827895,  0.05226748,  0.01714141, ..., -0.005661  ,
         0.02436945,  0.01169135],
       [-0.03512175,  0.0176127 ,  0.02929639, ...,  0.0022374 ,
         0.01826224,  0.03047828],
       [-0.00179529, -0.01935859,  0.02216526, ..., -0.01494863,
         0.04285045, -0.02584894]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.0408502, 0.9952368, 0.9868272, ..., 1.110908 , 1.0122148,
        1.0305483]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': array([ 8.5014108e-05, -3.6904723e-03, -2.2362696e-03, ...,
       -2.6434378e-03, -4.6255854e-03, -2.7873858e-03], dtype=float32)
                    },
                    'linear': {
                        'w': array([[-0.00317632,  0.01268681, -0.02180147, ..., -0.03509863,
         0.02900657,  0.05286129],
       [ 0.03748895, -0.01666473,  0.04159482, ...,  0.01672644,
        -0.03867533,  0.03111953],
       [-0.04813586, -0.03120456,  0.02479169, ...,  0.01334335,
        -0.04884216,  0.00229996],
       ...,
       [-0.03854724,  0.0137933 , -0.01018869, ..., -0.0093796 ,
         0.03359092,  0.00674478],
       [ 0.0059021 , -0.00476607, -0.01634962, ...,  0.02304816,
        -0.01485625,  0.01752446],
       [-0.02087662,  0.00162849,  0.0099754 , ..., -0.06226157,
         0.02198281, -0.00740848]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.0201908, 1.0224168, 1.0201058, ..., 1.0270991, 1.013838 ,
        1.0606363]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': array([ 0.00631881,  0.01150341,  0.00914302, ..., -0.0200112 ,
       -0.00163537, -0.000495  ], dtype=float32),
                    'scale': array([-0.03739248, -0.0243671 , -0.02632998, ..., -0.01486455,
        0.01445367,  0.04890316], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': array([1.2228152 , 1.1309394 , 1.2290405 , ..., 0.96604544, 0.9487428 ,
       0.9532705 ], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': array([[-6.4075930e-06,  4.2504727e-04, -1.1618993e-03, ...,
        -1.7168164e-03,  2.2836959e-04, -8.8550907e-05],
       [ 3.1805394e-04,  3.4957723e-04,  5.1292509e-04, ...,
         2.6313388e-03,  1.0807015e-03, -2.1224995e-03],
       [ 9.5978525e-04, -3.6766953e-03, -1.1368678e-03, ...,
         1.4873255e-03, -2.3283053e-03, -4.0190420e-03],
       ...,
       [-1.6945685e-03, -1.5151855e-03, -1.4960926e-03, ...,
         1.8141167e-03, -2.3590028e-03, -2.6155929e-03],
       [-5.4188799e-03,  4.6130762e-04, -6.1691484e-05, ...,
         3.2484343e-03, -2.7115741e-03, -3.6162001e-04],
       [-2.8124801e-03,  1.5307369e-04, -2.2465425e-05, ...,
         9.6123409e-04, -2.0123613e-03, -4.0718642e-05]], dtype=float32),
                    'w': array([[[-2.00603511e-02,  5.75137278e-03, -8.72080214e-03, ...,
         -2.70311963e-02,  1.24247791e-03, -2.36301706e-03],
        [ 3.99277313e-03,  2.94813123e-02, -1.99544337e-02, ...,
          1.95915089e-03,  1.23697640e-02,  5.81120653e-03],
        [-1.01987063e-03,  3.53949121e-03, -6.74259616e-03, ...,
          3.45695938e-04, -3.17200739e-03,  1.25762047e-02],
        ...,
        [ 1.12090185e-02,  5.59417754e-02,  2.80567389e-02, ...,
          5.72427809e-02, -2.45896727e-02, -4.41132709e-02],
        [-2.39311568e-02,  1.08251497e-02,  1.98488291e-02, ...,
          2.62170825e-02, -1.07316952e-02,  7.13963956e-02],
        [-5.38694439e-03,  2.10208986e-02,  4.16014344e-03, ...,
          1.06410738e-02,  1.50726573e-03, -1.23633593e-02]],

[[ 3.49556189e-03,  3.44262924e-04,  2.85613537e-03, ...,
         -1.43942228e-02, -5.75408980e-04,  3.76027485e-04],
        [ 8.77154060e-03,  1.46433450e-02, -5.25063137e-03, ...,
         -3.58833029e-04,  2.95653217e-03, -3.85825592e-03],
        [ 2.57922034e-03, -1.56476088e-02, -1.68384165e-02, ...,
          1.31746242e-02, -7.18129473e-03, -8.73307057e-04],
        ...,
        [ 2.63401377e-03,  4.11511064e-02,  1.82539616e-02, ...,
          3.93974334e-02, -2.09947210e-02, -4.63477969e-02],
        [-3.26764248e-02,  4.17060219e-03,  1.78119279e-02, ...,
          3.88964377e-02, -1.26539385e-02,  5.79634905e-02],
        [ 3.92154464e-03,  1.28147583e-02, -5.92348725e-03, ...,
          1.19894780e-02,  4.40148357e-03, -1.42792449e-03]],

[[-5.62313665e-03, -2.09410232e-03, -9.82910208e-03, ...,
         -3.35871242e-02, -2.54542637e-03, -3.40952398e-03],
        [ 5.54407481e-03,  1.78536046e-02, -7.02433800e-03, ...,
         -8.23897962e-03,  6.65213913e-03, -1.49944180e-03],
        [ 1.13043087e-02, -1.39708715e-02, -1.11245923e-02, ...,
          1.64344721e-02, -8.75243638e-03,  1.43328554e-03],
        ...,
        [-1.51386904e-03,  5.43856286e-02,  5.52712865e-02, ...,
          5.97689189e-02, -8.20651092e-03, -4.67697680e-02],
        [-2.99225654e-02, -6.27992325e-04,  1.96734313e-02, ...,
          5.00066131e-02, -2.26834826e-02,  6.30012304e-02],
        [ 7.33884191e-03,  7.73959141e-03,  2.24917941e-03, ...,
          1.09161446e-02, -6.46742759e-03, -3.35140945e-03]],

...,

[[ 1.10051483e-02, -1.85540915e-02,  9.35962051e-03, ...,
          6.85161399e-03,  5.54813165e-03, -6.58737845e-04],
        [-7.81041523e-03,  3.98438890e-03, -1.66073244e-03, ...,
         -1.19862985e-03, -1.22908978e-02, -6.26032334e-03],
        [ 7.88267050e-03, -2.79578473e-02, -1.36419181e-02, ...,
          2.40753926e-02, -1.45410048e-02, -1.76570639e-02],
        ...,
        [-6.93083834e-03,  1.26243681e-02, -1.05324322e-02, ...,
         -1.96885108e-03,  1.16551239e-02, -1.18715111e-02],
        [-6.10861601e-03,  3.95314310e-05,  1.73927583e-02, ...,
         -8.47204216e-03,  1.13081449e-04,  2.14985292e-03],
        [-2.08475161e-03, -1.37468986e-02,  5.63349109e-03, ...,
         -1.12989545e-02, -8.50978866e-03,  1.94774184e-03]],

[[ 2.74757389e-04, -4.94607608e-04,  6.77777128e-03, ...,
         -1.00600952e-02,  8.12825561e-03, -5.75583940e-03],
        [ 9.11341235e-03, -1.28179893e-03, -5.31734154e-03, ...,
          1.71367656e-02, -1.20344115e-02, -4.43551084e-03],
        [ 1.83838531e-02, -1.47979856e-02, -1.40168304e-02, ...,
          1.32685406e-02, -2.14968957e-02, -1.05311433e-02],
        ...,
        [-1.04027316e-02,  8.37995019e-03, -1.30986366e-02, ...,
          4.06161277e-03,  8.50717258e-03, -1.61217351e-03],
        [ 4.98227309e-03, -1.39553854e-02,  4.06156154e-03, ...,
         -7.87476171e-03,  5.47154248e-03, -6.78235898e-03],
        [-1.80154722e-02, -3.77116865e-03,  8.45899712e-03, ...,
         -1.32837438e-03, -1.42556825e-03, -1.07409863e-03]],

[[-1.08604589e-02,  1.28442254e-02,  5.53134922e-03, ...,
         -5.09914523e-03, -1.28980596e-02,  1.25545133e-02],
        [-1.18277026e-02,  8.53359420e-03,  3.05717532e-03, ...,
          1.41720111e-02, -1.60324126e-02,  1.07069116e-03],
        [-1.69687124e-03,  8.45217053e-03,  6.06904272e-03, ...,
         -1.82976536e-02,  7.91715831e-03,  2.23788200e-03],
        ...,
        [-6.64934842e-03,  6.04555802e-03, -1.44436248e-02, ...,
         -7.54461996e-03,  1.34342900e-02,  1.26932003e-03],
        [ 8.17108434e-03,  2.05422216e-03,  1.13008972e-02, ...,
         -1.15924431e-02,  7.23493379e-03, -1.01354849e-02],
        [-3.08707799e-03, -2.84547848e-03,  5.24008553e-03, ...,
         -9.74707212e-03, -9.47374292e-03,  8.77883460e-04]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.29073346, 0.29685742, 0.3037179 , ..., 0.34602788,
         0.29602417, 0.30575013],
        [0.32391438, 0.33644444, 0.3429411 , ..., 0.43286625,
         0.34944308, 0.33880782],
        [0.37621805, 0.43590567, 0.43363446, ..., 0.4849415 ,
         0.4190295 , 0.3955025 ],
        ...,
        [0.63017344, 0.85228944, 0.5222197 , ..., 0.42874113,
         0.574699  , 0.44997627],
        [0.53644234, 0.38935047, 0.48470864, ..., 0.5750971 ,
         0.32961357, 0.7605141 ],
        [0.37976965, 0.32443202, 0.3033245 , ..., 0.25471726,
         0.25815964, 0.25158453]]], dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': array([0.11543996, 0.19667041, 0.08449101, 0.15537748, 0.05297864,
       0.08272862, 0.08896066, 0.09299482, 0.14027727, 0.12275486,
       0.13754928, 0.11227061, 0.07794799, 0.11385844, 0.21155985,
       0.07013194, 0.11293491, 0.11814845, 0.14049305, 0.1071273 ,
       0.07294399, 0.11549649, 0.10932729, 0.03889112, 0.07322115,
       0.09397113, 0.08638889, 0.086013  , 0.09734396, 0.04915351,
       0.07681435, 0.0727135 , 0.06417551, 0.1102055 , 0.05094118,
       0.06759506, 0.10526094, 0.06561205, 0.08017935, 0.08598939,
       0.08295949, 0.10986121, 0.06075855, 0.06381149, 0.09573651,
       0.15352455, 0.10654029, 0.05720843, 0.1080655 , 0.08904675,
       0.11533722, 0.10753154, 0.14174831, 0.05750328, 0.10738507,
       0.13763388, 0.11135354, 0.07917753, 0.05568942, 0.18191479,
       0.19961604, 0.08848751, 0.1105781 , 0.11773358, 0.13192818,
       0.05453964, 0.07397411, 0.04283143, 0.09388805, 0.12555331,
       0.15708339, 0.14256129, 0.08874376, 0.08300523, 0.08548183,
       0.07495448, 0.01806459, 0.10216796, 0.08380181, 0.06248857],
      dtype=float32)
                },
                'post': {
                    'b': array([ 0.0038859 ,  0.00227933,  0.00126942, ..., -0.00807348,
       -0.0033168 , -0.00190115], dtype=float32),
                    'w': array([[[ 0.01001332,  0.00383801,  0.00365361, ..., -0.00576187,
          0.00987968,  0.00056312],
        [ 0.00502719,  0.0106297 , -0.00514524, ...,  0.00606945,
          0.00654817, -0.00433787],
        [ 0.00358599, -0.00257   ,  0.00474215, ..., -0.00819106,
          0.0009758 , -0.00469567],
        ...,
        [ 0.00208981,  0.01763259, -0.01958698, ...,  0.01154696,
         -0.009217  ,  0.00837894],
        [-0.0042141 , -0.00337535, -0.01742578, ..., -0.00651149,
          0.008242  ,  0.00044077],
        [-0.00331672,  0.00542616,  0.00861929, ...,  0.00442049,
         -0.00129945,  0.00199285]],

[[-0.00784615,  0.0007228 , -0.00331567, ..., -0.00073429,
          0.00443421,  0.00215626],
        [ 0.00290094,  0.00959504, -0.00379133, ..., -0.00251239,
          0.00683427, -0.00107421],
        [-0.00282467,  0.00305181, -0.0018921 , ..., -0.0041155 ,
          0.00290548, -0.00524714],
        ...,
        [-0.0083752 ,  0.00948612,  0.01147135, ...,  0.00237548,
         -0.00356866,  0.00163063],
        [-0.00435623, -0.02190368, -0.02925595, ..., -0.01722581,
         -0.01212152,  0.00048348],
        [-0.00665992, -0.00317996,  0.00328454, ...,  0.00392423,
          0.00951003, -0.00905988]],

[[-0.00525643,  0.0023299 , -0.00355807, ...,  0.00550958,
          0.00718864, -0.00195274],
        [ 0.00557126,  0.0047749 ,  0.0013244 , ...,  0.0020559 ,
          0.00676667,  0.0004616 ],
        [ 0.0036528 ,  0.00101903, -0.00275526, ..., -0.00442425,
         -0.00125036, -0.00322119],
        ...,
        [ 0.01212482,  0.00698769, -0.00469012, ...,  0.0119752 ,
         -0.01411841,  0.02146803],
        [ 0.00513848, -0.00179516, -0.0260424 , ..., -0.01782374,
         -0.00823828,  0.00242193],
        [-0.00726525, -0.00548198,  0.00631244, ...,  0.00095198,
          0.00166597, -0.01111033]],

...,

[[ 0.00183661,  0.00537889,  0.00282976, ...,  0.00681035,
         -0.00287456,  0.00020768],
        [-0.00195633, -0.01244443, -0.00306213, ..., -0.00138315,
          0.00965622, -0.00352222],
        [ 0.00638766, -0.00434901, -0.00251938, ...,  0.0082688 ,
          0.00016049,  0.00494107],
        ...,
        [-0.00437414, -0.00500601,  0.00331668, ..., -0.01888177,
          0.00244803, -0.00266423],
        [-0.01324576,  0.00685205, -0.00709169, ...,  0.00598535,
         -0.00090406, -0.00340358],
        [ 0.00550575,  0.00696265, -0.00663389, ...,  0.01093742,
         -0.0047209 , -0.00974895]],

[[-0.00350233, -0.00138139, -0.00094495, ..., -0.00453996,
         -0.00188128,  0.00299819],
        [ 0.00179204,  0.00947329, -0.00054177, ...,  0.00415827,
         -0.0054397 ,  0.00070148],
        [ 0.0055178 , -0.00677183, -0.00414257, ..., -0.00434643,
          0.00546411, -0.00096053],
        ...,
        [ 0.00150547, -0.01467462,  0.00350618, ..., -0.00134948,
          0.01102407,  0.00321508],
        [ 0.0170079 ,  0.00656268, -0.00057632, ...,  0.00752047,
         -0.00078745,  0.00861916],
        [ 0.00960819, -0.00827104, -0.00167912, ...,  0.00267513,
         -0.00212967,  0.00015723]],

[[ 0.00066472, -0.00265332, -0.00420681, ...,  0.00356282,
         -0.00272015, -0.00122229],
        [-0.00913083, -0.00096118, -0.00214067, ..., -0.00021383,
         -0.00040713, -0.0020092 ],
        [-0.0020909 ,  0.00057581, -0.00198702, ..., -0.00334667,
         -0.00107645, -0.00010455],
        ...,
        [-0.0086949 , -0.00821191,  0.00208952, ..., -0.00145943,
         -0.01484594, -0.00135062],
        [ 0.00198201,  0.000815  ,  0.00522504, ...,  0.00207165,
          0.0071472 , -0.00770016],
        [ 0.01279963,  0.00408498, -0.00312828, ...,  0.00053384,
         -0.00386579, -0.00476406]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.19123395, 0.14074375, 0.1416127 , ..., 0.1547206 ,
         0.1719081 , 0.14288525],
        [0.22511503, 0.2581373 , 0.23838745, ..., 0.2574504 ,
         0.24413465, 0.26848358],
        [0.15661407, 0.1571259 , 0.16474354, ..., 0.13923596,
         0.13650616, 0.15014726],
        ...,
        [0.31857383, 0.30543646, 0.28746378, ..., 0.27137437,
         0.2923366 , 0.27726355],
        [0.2973705 , 0.2631207 , 0.25538802, ..., 0.27492747,
         0.28831685, 0.28475752],
        [0.26332352, 0.26362932, 0.2317942 , ..., 0.22226565,
         0.2665483 , 0.24287087]]], dtype=float32)
                },
                'query': {
                    'b': array([[-0.05727649,  0.04552313, -0.04670092, ...,  0.00146665,
         0.02421395, -0.00091885],
       [-0.0103084 ,  0.00550128, -0.01701137, ...,  0.02177138,
         0.01786684,  0.00672262],
       [-0.19440088,  0.23860118,  0.19767429, ..., -0.2909671 ,
         0.26273695,  0.06778763],
       ...,
       [-0.01284689, -0.00560991, -0.00838155, ..., -0.00923357,
         0.00546553,  0.01274918],
       [-0.04595913,  0.00850777, -0.00376768, ...,  0.00486895,
        -0.06195865, -0.02272983],
       [-0.07012997, -0.08104558,  0.06873379, ...,  0.04658027,
        -0.02241261,  0.02544953]], dtype=float32),
                    'w': array([[[-1.8535994e-02,  9.5346302e-04, -1.9991446e-02, ...,
          1.0161528e-02, -1.9445473e-02,  2.0938339e-02],
        [ 4.7027450e-02, -1.9295773e-02,  5.2923034e-03, ...,
         -1.1656388e-02,  6.4211138e-02,  8.2570147e-03],
        [-7.1002268e-03,  1.9432401e-02,  3.4478415e-02, ...,
         -2.4406184e-02,  8.6701987e-03,  2.6081081e-02],
        ...,
        [ 8.1564359e-02,  9.4508991e-02,  9.0464605e-03, ...,
          1.5598202e-02, -5.1820800e-02, -2.5631530e-02],
        [ 8.2158990e-02, -1.8991157e-02, -4.1691676e-02, ...,
         -4.6170041e-02,  3.6681116e-02, -4.2611934e-02],
        [-8.7982453e-03, -5.2305171e-03,  2.0775546e-03, ...,
         -3.1847455e-02, -2.3901056e-02,  2.6486890e-02]],

[[-1.8576467e-02,  8.5971802e-03, -1.9604284e-02, ...,
          4.1900217e-03,  7.2023519e-03,  1.5307357e-02],
        [ 8.4553892e-03, -1.5333053e-02,  1.1094856e-02, ...,
         -2.2008531e-03,  3.6183935e-02, -8.0536548e-03],
        [ 6.3647947e-04,  2.3599932e-02,  2.9660890e-02, ...,
         -1.9744297e-02,  2.0901676e-02,  1.1683471e-02],
        ...,
        [ 6.4568944e-02,  8.3643161e-02,  1.1189053e-02, ...,
          2.0544671e-02, -5.1902432e-02, -1.3179816e-02],
        [ 8.4815398e-02, -5.9528677e-03, -3.0726269e-02, ...,
         -5.4053325e-02,  2.0459246e-02, -2.3801735e-02],
        [-6.9145137e-03, -1.0658662e-02,  2.6543209e-02, ...,
         -1.8605150e-02, -2.5469976e-02,  3.2749582e-02]],

[[-6.6796737e-03,  4.0878244e-03, -1.5109884e-02, ...,
         -1.8557297e-02, -8.3989222e-03,  1.9515160e-02],
        [ 2.6753681e-02,  5.5092634e-03, -9.7639320e-05, ...,
         -1.6482458e-02,  4.7398698e-02, -7.3832017e-04],
        [-5.1337848e-03,  6.3418280e-03,  2.3368003e-02, ...,
         -1.6323408e-02,  2.7475895e-03,  1.3939449e-02],
        ...,
        [ 5.7961229e-02,  1.1082336e-01,  1.8057462e-02, ...,
          2.1232987e-02, -4.0269952e-02, -9.0204747e-03],
        [ 9.7470656e-02,  6.0731932e-03, -2.4925102e-02, ...,
         -7.1226001e-02,  4.1544825e-02, -3.4857236e-02],
        [ 1.8736666e-02, -1.1518041e-03,  1.4533965e-02, ...,
         -4.3398786e-02, -2.3378823e-02,  4.3798864e-02]],

...,

[[-9.1594533e-04, -2.7340355e-05, -3.3326736e-03, ...,
          6.0474426e-03,  1.3866210e-03,  1.0323765e-03],
        [ 5.3152745e-03, -2.0928690e-03,  4.8185184e-04, ...,
         -9.7881826e-03, -8.2429210e-03, -8.9726523e-03],
        [ 6.6877482e-03,  1.2642969e-02,  1.8947086e-03, ...,
         -4.1028569e-03,  5.4840632e-03, -1.5325028e-03],
        ...,
        [-4.0247887e-03, -5.9987577e-03, -1.3183828e-02, ...,
          5.7974774e-03, -6.5089655e-03, -8.4939273e-03],
        [ 6.1868881e-03,  3.6727139e-03, -6.1531812e-03, ...,
         -9.8158640e-04,  7.5293705e-03,  5.6704287e-03],
        [-7.4572638e-03,  1.4056416e-03, -9.3617421e-03, ...,
         -1.0610916e-02,  1.9434205e-03,  2.7023890e-04]],

[[ 3.0095861e-04, -1.5247799e-03,  2.0903742e-03, ...,
          4.0808483e-03,  3.0722921e-03, -1.5267686e-03],
        [-3.1580557e-03, -8.4996018e-06, -2.2289488e-03, ...,
         -6.9968030e-03, -2.2506592e-04, -1.0762604e-02],
        [-1.8868812e-03,  2.0903235e-02,  1.0260254e-02, ...,
         -7.1998122e-03,  5.1443158e-03,  1.2138507e-02],
        ...,
        [-2.5156606e-04,  8.2086452e-04, -7.4347402e-03, ...,
         -3.2056423e-03, -4.1695866e-03, -7.0315442e-04],
        [ 5.9153480e-03, -3.8115361e-03,  2.3869504e-03, ...,
         -1.0516641e-02,  3.5312949e-03, -1.9034272e-04],
        [ 1.1917896e-03,  5.0350041e-03,  3.0626596e-03, ...,
          4.8030242e-03, -2.8061287e-03, -1.4090024e-04]],

[[-2.9711812e-03, -3.1196219e-03,  3.0905388e-03, ...,
          5.2482258e-03, -7.5695748e-03, -9.2584523e-04],
        [-5.9177619e-03, -2.8969548e-04,  1.5838912e-03, ...,
          1.1437699e-02,  6.0875183e-03, -4.7865333e-03],
        [-2.0276653e-03,  6.7625120e-03,  6.7691165e-03, ...,
          2.1833940e-03,  4.4609946e-03,  5.0599249e-03],
        ...,
        [-7.3539508e-03, -6.0827681e-03, -1.7312258e-03, ...,
          1.2796396e-02, -9.6651735e-03,  1.4769682e-04],
        [-5.5717267e-03, -1.3548125e-03,  4.8010638e-03, ...,
         -6.7730816e-03, -5.1152851e-03, -5.0927578e-03],
        [ 2.5766164e-03,  1.0111049e-02, -4.2173183e-03, ...,
         -6.2474562e-03,  6.0498621e-03, -6.5683518e-03]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.4614331 , 0.45003605, 0.34651205, ..., 0.3119441 ,
         0.39296058, 0.32459667],
        [0.276341  , 0.2899145 , 0.43911663, ..., 0.49180484,
         0.3920436 , 0.41945332],
        [0.33191723, 0.8000541 , 0.48445222, ..., 0.65729636,
         0.45609203, 0.6975884 ],
        ...,
        [0.67445004, 0.6936323 , 0.5252021 , ..., 0.45358855,
         0.5806137 , 0.44342574],
        [0.7143098 , 0.38224772, 0.4478439 , ..., 0.58324504,
         0.37454212, 0.6270008 ],
        [0.32466984, 0.36532098, 0.2966742 , ..., 0.35533345,
         0.28476945, 0.33105844]]], dtype=float32)
                },
                'value': {
                    'b': array([[ 2.0189777e-02, -1.6526524e-02,  7.5444379e-03, ...,
        -5.4049934e-03, -2.8437520e-03, -6.9945725e-03],
       [-4.1467221e-03, -8.7834820e-03,  1.3447361e-03, ...,
        -8.4275985e-03,  6.2074391e-03,  1.4883006e-02],
       [-2.9682057e-02, -4.1251830e-04, -9.7222412e-03, ...,
         8.8203693e-04, -1.3974663e-02, -6.0880790e-03],
       ...,
       [-5.1026130e-03,  2.3056350e-03, -8.9430797e-04, ...,
        -2.3210514e-03, -1.1109073e-02, -8.0586029e-03],
       [-4.9333312e-03, -7.2803735e-03, -1.4136798e-02, ...,
         3.3284265e-03,  1.0361077e-02, -2.0654943e-02],
       [ 3.9697597e-06,  6.5383087e-03,  6.8615028e-04, ...,
        -7.1053160e-03, -5.0539179e-03,  1.7412959e-02]], dtype=float32),
                    'w': array([[[ 1.14159631e-02, -1.59800947e-02, -3.59197892e-03, ...,
          9.23145469e-03,  1.12693738e-02,  1.20775970e-02],
        [ 2.07490451e-03,  2.38946197e-03,  1.61821153e-02, ...,
         -1.75665040e-02, -1.09817199e-02,  1.10940356e-02],
        [-1.35025522e-02,  1.15187317e-02,  1.60536468e-02, ...,
          2.25036754e-03, -3.16550490e-03, -3.02010793e-02],
        ...,
        [-3.36391851e-02, -1.05752600e-02,  1.97917111e-02, ...,
          1.07368371e-02,  9.73035581e-04,  1.68171246e-02],
        [-4.20491706e-04, -1.12381661e-02, -3.82524543e-02, ...,
         -1.18914908e-02, -4.05434705e-03, -1.41844817e-03],
        [ 6.22682087e-03, -8.74834368e-04, -2.80750962e-03, ...,
          7.22780824e-03,  2.30072369e-03, -1.35360025e-02]],

[[ 4.45867516e-03, -4.11099382e-03, -1.15284566e-02, ...,
         -4.86745732e-04, -1.58392033e-03,  9.92201269e-03],
        [ 6.68235263e-03, -7.87907187e-03,  3.91036458e-03, ...,
         -1.22944952e-03,  3.23050632e-03, -1.35624027e-02],
        [ 1.12104090e-03,  8.19841772e-03,  1.03040989e-02, ...,
          8.91722366e-03, -1.94189418e-03, -8.92079333e-05],
        ...,
        [-2.66050510e-02, -4.14850470e-03,  2.08170200e-03, ...,
          4.12974786e-03, -1.79608993e-03,  5.54801140e-04],
        [ 6.62425999e-03, -6.20562816e-03, -1.99722536e-02, ...,
         -1.20812552e-02, -1.05366986e-02, -7.08280876e-03],
        [-2.08399899e-04,  4.79412684e-03, -3.73588118e-04, ...,
          1.42460875e-02, -2.64002098e-04, -1.68110225e-02]],

[[ 1.27272296e-03,  3.88708035e-03, -1.40979048e-02, ...,
          3.12086777e-04, -1.10901827e-02,  2.06127912e-02],
        [-2.02940241e-03, -2.15654951e-02,  1.58768129e-02, ...,
         -1.60188496e-03,  1.24952262e-02,  6.66512083e-03],
        [-1.12345326e-03,  1.40307182e-02,  2.83075999e-02, ...,
          1.21791130e-02,  2.09182352e-02, -2.54418571e-02],
        ...,
        [-3.45040634e-02, -2.09277449e-03,  4.57704300e-03, ...,
          1.86158414e-03,  1.51762459e-02,  1.68939331e-03],
        [-4.19804174e-03, -1.21910796e-02, -2.39743795e-02, ...,
         -1.22641986e-02, -4.19842021e-04, -8.42225458e-03],
        [ 4.47197119e-03, -1.19095540e-03, -3.47271608e-03, ...,
          1.76485498e-02, -8.38476419e-03, -9.54039767e-03]],

...,

[[ 4.26548254e-03,  2.08089338e-03,  1.00391626e-03, ...,
         -6.46455726e-03,  7.72333378e-03,  6.29935972e-03],
        [-6.97384635e-03, -9.70582385e-03,  6.58419356e-03, ...,
         -6.03764085e-03, -5.26762009e-03, -7.28255464e-03],
        [-1.02760049e-03,  8.02699535e-04,  2.19186069e-03, ...,
         -6.15477678e-04,  3.07763438e-03,  7.58022256e-03],
        ...,
        [-4.05637547e-03, -1.17339231e-02, -1.19129603e-04, ...,
          4.90481313e-03,  4.83122142e-03, -8.76050629e-03],
        [-4.03867947e-04, -1.60734099e-03, -4.01273230e-03, ...,
          7.93322863e-04,  1.97835849e-03,  1.00982864e-03],
        [-3.57880420e-03, -1.37891318e-03, -7.47711631e-03, ...,
          1.04652047e-02, -5.84660564e-03,  1.55211147e-03]],

[[ 6.78349659e-03,  8.07991717e-03, -3.03134602e-03, ...,
          7.22206780e-04,  2.69830902e-03,  6.43080391e-04],
        [-4.21012333e-03, -2.72593298e-03, -6.69471733e-03, ...,
          3.75081389e-03, -1.31997308e-02,  3.86104337e-03],
        [-6.55550859e-04,  3.54344887e-03,  3.86863761e-03, ...,
         -5.34136023e-04, -3.87167349e-03, -6.13696873e-03],
        ...,
        [ 1.65994873e-03,  6.80680445e-04,  3.95616749e-03, ...,
         -1.70886738e-03, -9.31852404e-03, -7.05520343e-03],
        [ 9.07271914e-03, -3.48147936e-03,  1.69612555e-04, ...,
          9.67306551e-03,  8.45150743e-03,  3.43226548e-03],
        [-8.28925928e-04,  7.17098126e-04,  2.70726462e-03, ...,
          9.16007534e-03,  1.16995454e-03,  7.09785149e-03]],

[[-3.12572159e-03, -4.58090147e-03,  4.04954772e-04, ...,
         -4.26897453e-03,  1.00395828e-03,  3.13942088e-04],
        [ 6.21284498e-03,  1.91911927e-03,  6.82118209e-03, ...,
          1.77347346e-03,  3.33468284e-04, -1.69885426e-03],
        [ 6.59609400e-03, -2.01542350e-03,  5.82617614e-03, ...,
         -6.56293333e-03,  4.57923766e-03, -3.00541869e-03],
        ...,
        [ 5.23311552e-03,  5.85666252e-03,  1.00663514e-03, ...,
          4.23181895e-03, -4.45733871e-03,  5.47718396e-03],
        [ 2.15847930e-03, -2.96204607e-03,  6.17724471e-03, ...,
         -5.99337555e-03,  2.74075102e-03, -4.38916869e-03],
        [ 5.89331030e-04, -4.49438952e-03,  1.91602285e-03, ...,
          2.74458597e-03,  5.07364632e-04,  1.85802171e-03]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.19661559, 0.2062873 , 0.19937374, ..., 0.23163205,
         0.21414809, 0.20715529],
        [0.3326665 , 0.36796087, 0.3426592 , ..., 0.41867024,
         0.32002532, 0.40541944],
        [0.2070302 , 0.19507912, 0.24107307, ..., 0.16693698,
         0.21783899, 0.20636791],
        ...,
        [0.363433  , 0.40726158, 0.3263816 , ..., 0.35479748,
         0.3959102 , 0.38188574],
        [0.35043123, 0.23056264, 0.35292384, ..., 0.3600547 ,
         0.2950996 , 0.43130183],
        [0.19793825, 0.24577667, 0.29478887, ..., 0.2744211 ,
         0.2816329 , 0.27519068]]], dtype=float32)
                }
            }
        },
        'x_layers_1': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': array([ 0.00140202,  0.0150145 ,  0.01619885, ..., -0.0080475 ,
        0.00246115,  0.00246259], dtype=float32)
                    },
                    'linear': {
                        'w': array([[-0.00787557,  0.01338205,  0.02889786, ..., -0.0941308 ,
         0.03565976, -0.01046602],
       [ 0.05106761,  0.04127442, -0.02096361, ..., -0.10027318,
        -0.02925511,  0.03009592],
       [ 0.05306583, -0.0506343 ,  0.01457519, ..., -0.09564968,
         0.04032068,  0.03279277],
       ...,
       [ 0.00158152,  0.00071211, -0.00561694, ...,  0.02137044,
         0.03315446,  0.03018216],
       [-0.04751755,  0.0064413 ,  0.02847996, ..., -0.04394938,
         0.02009094,  0.03739234],
       [ 0.05406936, -0.04232696, -0.0204008 , ...,  0.02417432,
        -0.05176357,  0.01570461]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.250599  , 1.3255891 , 1.1087048 , ..., 1.1848813 , 1.0188245 ,
        0.96637166]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': array([ 0.00089951, -0.00035887, -0.00030018, ...,  0.00080614,
       -0.00512187, -0.00519201], dtype=float32)
                    },
                    'linear': {
                        'w': array([[-0.0554122 ,  0.01932358, -0.03726121, ...,  0.01832989,
         0.00719377, -0.00606497],
       [ 0.06329854,  0.01490471, -0.02579143, ...,  0.02443456,
        -0.00130729,  0.01079406],
       [ 0.0415604 ,  0.02598145, -0.04248028, ..., -0.00492932,
        -0.04967967, -0.0337037 ],
       ...,
       [-0.01317128, -0.02745201, -0.01492178, ..., -0.07122612,
        -0.0111488 ,  0.05783502],
       [ 0.03689443,  0.02630216, -0.00501244, ..., -0.00371938,
        -0.0267868 , -0.00740327],
       [-0.02904673, -0.04345071, -0.04526708, ..., -0.01844003,
        -0.00169297, -0.02380434]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.0208927, 1.0823526, 1.06109  , ..., 1.0328333, 1.0591458,
        1.056273 ]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': array([ 0.00193563,  0.00100008, -0.00270318, ..., -0.01044109,
       -0.00056009,  0.00339484], dtype=float32),
                    'scale': array([-0.0271673 , -0.02952997, -0.02053728, ..., -0.02958241,
        0.01236532, -0.00586465], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': array([1.083301 , 1.0569358, 1.0748862, ..., 0.9615119, 1.041255 ,
       1.0239404], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': array([[ 0.00106008,  0.00027187,  0.00024831, ..., -0.00085115,
        -0.00164053,  0.00174351],
       [-0.00210733, -0.00113148,  0.00045784, ...,  0.00069043,
        -0.00029879,  0.00441118],
       [ 0.00184593, -0.00026972,  0.00052276, ..., -0.00254204,
         0.00205951,  0.00068552],
       ...,
       [ 0.0012805 , -0.00020245,  0.0004382 , ..., -0.00219003,
         0.00275843,  0.0028203 ],
       [-0.0011356 , -0.00077106, -0.00172974, ...,  0.00030115,
        -0.0012361 , -0.00013721],
       [-0.0013096 ,  0.00173459, -0.00126198, ...,  0.00091986,
        -0.00124292, -0.00322954]], dtype=float32),
                    'w': array([[[-1.80063285e-02, -5.09520667e-03,  5.54195978e-03, ...,
         -4.16026497e-03, -1.01513192e-02, -5.73090138e-03],
        [ 1.24108866e-02, -2.54758168e-03, -8.32659949e-04, ...,
          2.82166386e-03, -2.69944640e-03, -1.34731792e-02],
        [ 9.65868309e-03,  2.00738781e-03,  4.31240816e-03, ...,
         -4.11521178e-03,  6.59295451e-03,  3.14612966e-03],
        ...,
        [ 8.63340031e-03, -1.49822459e-02,  2.24748831e-02, ...,
          1.36522716e-02, -5.36048599e-03, -9.77457385e-04],
        [-2.86621023e-02,  3.08751990e-03,  1.16948429e-02, ...,
         -1.53233688e-02, -4.77650156e-03, -4.27124649e-02],
        [ 2.98779812e-02, -1.83865577e-02, -2.87840590e-02, ...,
         -9.77942161e-03, -2.34802477e-02, -2.39428822e-02]],

[[-1.33668408e-02, -1.00676240e-02, -5.70136961e-03, ...,
         -1.66382603e-02, -1.74216402e-03,  3.86234443e-03],
        [-3.20761302e-03, -1.45851942e-02, -7.40802940e-03, ...,
          5.73277427e-03,  1.47106219e-02, -1.01560140e-02],
        [ 6.47455594e-03,  4.07713186e-03,  1.73721090e-02, ...,
         -5.47811575e-03, -4.68625408e-03,  5.13200369e-03],
        ...,
        [ 1.97917577e-02, -3.39834690e-02,  8.14834330e-03, ...,
          2.17960728e-03, -8.99836142e-03, -6.62191678e-03],
        [-2.15294003e-03, -1.15637388e-03,  4.53943666e-03, ...,
         -1.69788916e-02, -6.66009635e-03, -3.63575034e-02],
        [ 3.43549661e-02, -2.83050705e-02, -2.69499645e-02, ...,
         -5.87242190e-03, -2.92898305e-02, -1.18332729e-02]],

[[-2.85892207e-02, -1.46722980e-02,  7.81959016e-03, ...,
         -1.23542277e-02,  1.08651072e-02,  9.34736617e-03],
        [ 1.05932048e-02,  1.99711937e-02,  6.66643842e-04, ...,
          3.93575709e-03, -6.51192293e-03, -5.11098700e-03],
        [ 3.51419710e-02, -2.70137955e-02,  1.54421236e-02, ...,
         -1.45175948e-03, -2.48372066e-03,  5.88282244e-03],
        ...,
        [ 1.99931581e-02, -2.40505375e-02,  1.46369664e-02, ...,
          1.28342500e-02, -1.90169574e-03, -1.07531007e-02],
        [-2.11218074e-02, -4.18915786e-03,  4.71540028e-03, ...,
         -7.91548379e-03, -9.07850452e-04, -3.32458057e-02],
        [ 2.43409388e-02, -1.53097166e-02, -1.62991192e-02, ...,
         -8.76943395e-03, -1.77606437e-02, -3.73071767e-02]],

...,

[[-1.27441259e-02, -4.23898315e-03, -2.27499120e-02, ...,
          1.20623074e-02, -1.26639400e-02, -1.58614237e-02],
        [-1.85966250e-02,  1.59245767e-02, -3.53435008e-03, ...,
          1.40544707e-02, -1.80466089e-03, -4.52811364e-03],
        [ 3.21914908e-03, -1.93545688e-02, -2.52388138e-03, ...,
          8.44706316e-04, -7.61902798e-03,  1.67526845e-02],
        ...,
        [-7.61939818e-03, -7.76077993e-03, -1.14548835e-03, ...,
         -1.71117159e-03,  5.18547883e-03,  6.08849106e-03],
        [-3.20350780e-04, -1.29192844e-02, -1.91536620e-02, ...,
         -8.29553232e-03,  1.62451826e-02,  7.16950139e-03],
        [-9.32032336e-03,  1.93152633e-02,  5.65178646e-03, ...,
          3.33583131e-02,  2.75431434e-03, -1.55268377e-02]],

[[-5.68405725e-03,  8.43854435e-03, -1.76718794e-02, ...,
          7.80173484e-03,  4.15704533e-04, -6.32757042e-03],
        [ 1.93122390e-03, -1.71806868e-02, -9.17661018e-05, ...,
          3.01939389e-03,  5.51862118e-04, -9.69013141e-04],
        [ 5.46942092e-03,  3.15147685e-03,  1.49422586e-02, ...,
          6.87818183e-03, -8.79333168e-03, -1.42828710e-02],
        ...,
        [ 2.21097213e-03,  1.58943348e-02,  1.20991108e-03, ...,
          5.27463574e-03, -5.07480511e-03, -6.44097920e-04],
        [-1.50747597e-02,  3.40713821e-02,  1.34509697e-03, ...,
          5.09260874e-03,  3.12096020e-03, -2.22732755e-03],
        [-2.67531560e-03,  4.00075363e-03,  7.40008941e-03, ...,
         -4.11927095e-03,  1.26301218e-02, -4.01114253e-03]],

[[ 7.48981442e-03, -1.62659085e-03, -1.02963496e-03, ...,
         -8.87682289e-03,  9.19368118e-03,  1.30671617e-02],
        [-8.24701879e-03,  2.49495194e-03,  1.91343501e-02, ...,
         -2.36123893e-02, -4.69408277e-03,  4.21702489e-03],
        [-1.06736133e-02, -3.66469217e-03, -4.78547020e-03, ...,
          5.89408027e-03,  1.24248369e-02, -9.91146808e-05],
        ...,
        [ 9.79922898e-03,  4.19031530e-05, -2.21290975e-03, ...,
          1.33938035e-02, -1.72343049e-02, -4.91101015e-03],
        [ 5.48506901e-03,  1.00362059e-02, -1.10709118e-02, ...,
         -1.10125998e-02, -9.97462124e-03,  7.39515852e-03],
        [-2.61746198e-02,  3.75667843e-03, -2.53216662e-02, ...,
          2.32830588e-02, -1.51165668e-02, -2.59454325e-02]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.38537782, 0.36555395, 0.44076756, ..., 0.38749555,
         0.40258914, 0.39220354],
        [0.37189114, 0.49009037, 0.37673765, ..., 0.42121136,
         0.42172438, 0.37394658],
        [0.4474    , 0.505426  , 0.4788912 , ..., 0.42287984,
         0.39158925, 0.44060087],
        ...,
        [0.36623842, 0.42257005, 0.43054697, ..., 0.32004422,
         0.32337335, 0.3952168 ],
        [0.4860463 , 0.44154626, 0.42089644, ..., 0.4159828 ,
         0.4152751 , 0.45153633],
        [0.43383196, 0.41150486, 0.45498753, ..., 0.4020867 ,
         0.4382626 , 0.39161322]]], dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': array([0.0840179 , 0.08244094, 0.06245576, 0.08689576, 0.08572489,
       0.08779805, 0.07697938, 0.09394022, 0.08097968, 0.08754763,
       0.08341486, 0.05427131, 0.08808171, 0.07731166, 0.0860716 ,
       0.07904376, 0.07062105, 0.07418054, 0.07284027, 0.07907584,
       0.06827318, 0.07395329, 0.08116236, 0.0874623 , 0.08157628,
       0.0872488 , 0.06434779, 0.06768143, 0.0991152 , 0.07790367,
       0.0773618 , 0.07666542, 0.07887019, 0.08864841, 0.05583115,
       0.09895247, 0.10453542, 0.12288661, 0.11304845, 0.1039682 ,
       0.06304675, 0.10898725, 0.1052065 , 0.07074126, 0.07930253,
       0.09781079, 0.12111677, 0.09354454, 0.08050405, 0.09708615,
       0.09194972, 0.08502522, 0.07665642, 0.09167423, 0.05380047,
       0.07951675, 0.10532993, 0.0631919 , 0.0652303 , 0.09186361,
       0.0613084 , 0.09283226, 0.10487343, 0.11428737, 0.05996763,
       0.07991926, 0.07705473, 0.08530728, 0.0627922 , 0.10198504,
       0.08804259, 0.0772656 , 0.12166466, 0.07498417, 0.0551027 ,
       0.07845861, 0.0941859 , 0.08346103, 0.06084656, 0.0929875 ],
      dtype=float32)
                },
                'post': {
                    'b': array([-8.5084306e-05, -1.1306409e-03, -3.8026206e-04, ...,
        8.4014353e-04, -2.9504241e-03, -2.7141783e-03], dtype=float32),
                    'w': array([[[-0.00708895, -0.01480758,  0.01334359, ..., -0.01726026,
          0.00060817, -0.00494491],
        [ 0.01220738,  0.00202692, -0.00111894, ..., -0.00664444,
          0.0240082 ,  0.00878899],
        [ 0.00194961, -0.01813553,  0.00442683, ..., -0.02262122,
         -0.01098372,  0.00288465],
        ...,
        [-0.004893  , -0.00926619, -0.00032269, ...,  0.00613354,
          0.01687936,  0.00583418],
        [ 0.00492367,  0.00383521, -0.00515759, ...,  0.00415317,
          0.00398865, -0.00464302],
        [ 0.00809234,  0.00042906,  0.01076343, ...,  0.00149677,
          0.00985494, -0.00063159]],

[[ 0.00246185, -0.00765722,  0.00894078, ..., -0.01340171,
         -0.01079085, -0.0011629 ],
        [ 0.00147542, -0.00834893,  0.01121476, ...,  0.01543822,
          0.00342024, -0.00650793],
        [-0.00200861, -0.00277726,  0.01831196, ..., -0.00682348,
         -0.02272334, -0.01270596],
        ...,
        [-0.00428783, -0.00712438, -0.00502129, ...,  0.00878618,
          0.00305978,  0.02874406],
        [ 0.00445584, -0.00468032,  0.0072703 , ...,  0.01272799,
         -0.01477935,  0.01846482],
        [ 0.00147231, -0.02466133,  0.01886973, ..., -0.00191433,
          0.00757758, -0.01421218]],

[[-0.00514883,  0.01132185, -0.00115655, ..., -0.00631013,
         -0.00804404,  0.01609146],
        [-0.0072751 , -0.00240126,  0.00546431, ...,  0.00898082,
         -0.0069311 , -0.00859138],
        [ 0.00603159,  0.00096185,  0.01038047, ..., -0.01199295,
         -0.00993051,  0.00589456],
        ...,
        [ 0.00329012, -0.00998469,  0.00355849, ...,  0.00554496,
          0.01540993,  0.01477036],
        [ 0.00168463, -0.00042731,  0.00468066, ...,  0.01656279,
         -0.01535067,  0.00802685],
        [-0.00103584,  0.00505322,  0.02339929, ..., -0.01299549,
          0.00252153, -0.01547725]],

...,

[[-0.00737087,  0.00111887,  0.00770168, ..., -0.00085506,
          0.00704482, -0.00228771],
        [ 0.00897177, -0.01145018, -0.00070282, ...,  0.00570436,
         -0.00303456,  0.00578517],
        [-0.00020333, -0.00843141,  0.01788862, ...,  0.00169104,
         -0.00016737, -0.00554323],
        ...,
        [ 0.00566079,  0.0148934 ,  0.00826112, ..., -0.00828986,
         -0.00548168,  0.00413958],
        [-0.00321849, -0.00664052, -0.00179358, ...,  0.00676621,
          0.00398269,  0.00728607],
        [-0.0150725 ,  0.01272176,  0.00335532, ...,  0.00981382,
         -0.00203804, -0.01907342]],

[[ 0.00516852, -0.00299152,  0.00561673, ..., -0.00392041,
          0.01006778, -0.01084407],
        [ 0.00782847,  0.01425212, -0.00375229, ..., -0.00274928,
         -0.00041469, -0.00183591],
        [ 0.0058049 ,  0.00619497, -0.00545526, ..., -0.00565154,
          0.01369412,  0.00989184],
        ...,
        [ 0.00081749,  0.00748981, -0.01918131, ...,  0.00990216,
          0.00938182,  0.00307292],
        [-0.00162893, -0.00860842, -0.00685418, ..., -0.01229144,
         -0.00692623,  0.01290068],
        [-0.00142746,  0.0060008 ,  0.00480887, ...,  0.00169146,
          0.01177845, -0.01732686]],

[[ 0.00828316, -0.00220981,  0.00527866, ...,  0.00705342,
          0.01052925, -0.00860018],
        [ 0.00089691, -0.00227484, -0.00601018, ..., -0.00764392,
         -0.00332279, -0.00458629],
        [ 0.00742584, -0.00383384,  0.00629323, ...,  0.00774965,
         -0.00140316, -0.00998758],
        ...,
        [-0.01038045,  0.0074235 , -0.00076588, ...,  0.00299953,
          0.01173578, -0.00073879],
        [-0.00396501, -0.00384969, -0.00263106, ...,  0.0007746 ,
         -0.00322374,  0.00686969],
        [-0.00165319, -0.00069949, -0.00341704, ...,  0.00016102,
          0.00203316, -0.00223182]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.26153123, 0.25561446, 0.24397352, ..., 0.27744117,
         0.22440472, 0.28168353],
        [0.2576756 , 0.29397646, 0.27972355, ..., 0.28257143,
         0.3010583 , 0.28558657],
        [0.30174646, 0.2668868 , 0.283249  , ..., 0.2837114 ,
         0.29388875, 0.28595865],
        ...,
        [0.29271272, 0.3018412 , 0.31280938, ..., 0.27421346,
         0.2876453 , 0.31825015],
        [0.28015548, 0.25546336, 0.31681937, ..., 0.26992685,
         0.28597566, 0.28775772],
        [0.29379869, 0.31015328, 0.29396722, ..., 0.3012949 ,
         0.30367255, 0.2926929 ]]], dtype=float32)
                },
                'query': {
                    'b': array([[ 0.01952027,  0.02592703,  0.01507241, ...,  0.01695433,
        -0.00907723,  0.00838097],
       [-0.01057168, -0.00455518, -0.01358485, ...,  0.00340533,
         0.01626122, -0.01695308],
       [-0.00089808,  0.00056304, -0.02251254, ...,  0.0155376 ,
        -0.01954361,  0.0131553 ],
       ...,
       [ 0.0013416 ,  0.00240092,  0.00654386, ...,  0.02299946,
        -0.01865708, -0.00998419],
       [ 0.02600852, -0.0133097 , -0.0117526 , ...,  0.02061324,
         0.03217793,  0.0197717 ],
       [-0.00274182,  0.01528802, -0.01193944, ...,  0.02231876,
        -0.01095807, -0.01857167]], dtype=float32),
                    'w': array([[[-0.00167942,  0.01055444, -0.01093264, ..., -0.02210693,
          0.01847492, -0.00255125],
        [ 0.01091587, -0.00381827,  0.01870508, ..., -0.00639131,
         -0.0212311 , -0.002378  ],
        [ 0.01231089, -0.00437073, -0.01020596, ...,  0.05093126,
          0.01480385,  0.01470868],
        ...,
        [ 0.00866831,  0.01642605,  0.01773182, ...,  0.00787777,
          0.00646162,  0.0289838 ],
        [ 0.03137889,  0.01285845,  0.01455711, ..., -0.02560751,
          0.02575761, -0.00764195],
        [-0.01348505, -0.04771884, -0.00423431, ...,  0.00435013,
          0.0177009 , -0.0179799 ]],

[[ 0.00105582, -0.0048518 ,  0.01458317, ..., -0.00052341,
         -0.00440311,  0.01890587],
        [-0.00530806,  0.00574125, -0.00498638, ...,  0.00132487,
          0.00286455, -0.02257999],
        [ 0.00622075, -0.01728141,  0.00078462, ...,  0.07867897,
          0.00543411,  0.01140564],
        ...,
        [ 0.0082514 ,  0.00937226,  0.00842577, ...,  0.02527124,
         -0.01874645,  0.0192023 ],
        [ 0.02305375, -0.00536437,  0.01069765, ...,  0.0073627 ,
          0.0395656 , -0.01459153],
        [ 0.00405632, -0.04717059, -0.01289591, ...,  0.00814661,
          0.00738584, -0.03053756]],

[[-0.00470794,  0.01182036, -0.00831278, ...,  0.01435365,
         -0.01407391, -0.01395501],
        [ 0.00370709,  0.00258553, -0.00435809, ...,  0.00552407,
         -0.01145876, -0.0325379 ],
        [ 0.00480101, -0.01094179,  0.01678831, ...,  0.05895756,
         -0.0168597 ,  0.01058885],
        ...,
        [ 0.01283531, -0.01443544,  0.00780167, ..., -0.00460875,
          0.00241825,  0.02448686],
        [ 0.01687072,  0.01563838, -0.0047245 , ..., -0.01411427,
          0.01573402, -0.01434774],
        [ 0.0302384 , -0.06258623,  0.02342253, ..., -0.02170804,
          0.00516582, -0.03215941]],

...,

[[ 0.00813529,  0.01482299,  0.00575231, ...,  0.01250606,
         -0.00739157,  0.00974235],
        [-0.00804375,  0.00483053,  0.00173363, ...,  0.01357365,
         -0.00292534, -0.01122615],
        [ 0.00013886, -0.00906408, -0.01706935, ...,  0.00659653,
          0.00255185,  0.01143132],
        ...,
        [ 0.00647265, -0.0042616 , -0.00702474, ...,  0.00492873,
         -0.01319877, -0.00461267],
        [ 0.02027165,  0.00129452,  0.00409241, ..., -0.00074975,
         -0.00304098, -0.00438217],
        [-0.0072379 ,  0.00628982, -0.00515819, ...,  0.00915095,
          0.01885838,  0.00171674]],

[[ 0.0074124 ,  0.01073542, -0.00898895, ...,  0.00013246,
          0.02021744,  0.00313002],
        [ 0.00970622, -0.00732328, -0.00295661, ..., -0.00404823,
          0.00412052, -0.00195823],
        [ 0.0054835 ,  0.00488634, -0.0083871 , ..., -0.00971892,
         -0.02686961,  0.00529323],
        ...,
        [-0.00080711,  0.00698511, -0.00806548, ...,  0.00732281,
         -0.00373247,  0.00414027],
        [ 0.01257625, -0.01347495, -0.01232266, ...,  0.01663786,
         -0.01641294,  0.02857017],
        [-0.03199352,  0.01330567, -0.02327588, ...,  0.03041705,
         -0.00215055, -0.02280348]],

[[ 0.01646333,  0.00423958,  0.01651123, ..., -0.01476896,
          0.006426  ,  0.022834  ],
        [-0.00171471,  0.01116127,  0.00777352, ..., -0.0167604 ,
         -0.00763599, -0.01560871],
        [-0.00409038, -0.000114  , -0.02664903, ..., -0.0039424 ,
          0.01968246, -0.01381156],
        ...,
        [ 0.01327849, -0.03327335,  0.01283411, ...,  0.02662477,
         -0.01906407, -0.00113409],
        [ 0.02000252, -0.02245389, -0.00270883, ...,  0.01630181,
         -0.01032738,  0.01874355],
        [ 0.00188271, -0.002353  , -0.05625219, ...,  0.00055419,
         -0.04353327, -0.0223032 ]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.47950155, 0.50289047, 0.5207034 , ..., 0.45556846,
         0.6400834 , 0.52901435],
        [0.52863854, 0.4868401 , 0.46732205, ..., 0.5636472 ,
         0.5286924 , 0.5251258 ],
        [0.46756184, 0.5442631 , 0.5856073 , ..., 0.5593357 ,
         0.6385543 , 0.56340754],
        ...,
        [0.45784828, 0.44042316, 0.4172834 , ..., 0.441837  ,
         0.46731767, 0.41661546],
        [0.6531727 , 0.49530742, 0.54074895, ..., 0.5277548 ,
         0.6216696 , 0.5350829 ],
        [0.5234628 , 0.6732489 , 0.62287027, ..., 0.589735  ,
         0.52377564, 0.53240305]]], dtype=float32)
                },
                'value': {
                    'b': array([[-1.4370600e-03, -1.2042513e-03,  1.0407623e-03, ...,
         1.4411649e-03, -8.5053267e-04,  7.9507163e-06],
       [-8.6065737e-04, -2.0831903e-03,  2.4297652e-03, ...,
         1.1107546e-03,  9.7443461e-03,  1.0844832e-04],
       [ 2.4037904e-03, -1.5152757e-03, -1.1029590e-05, ...,
         2.4529195e-03,  2.2916619e-03,  3.8734211e-03],
       ...,
       [ 4.2531057e-03, -1.1110497e-03,  4.8523545e-03, ...,
        -5.3573218e-03, -1.9030088e-03, -7.4371318e-03],
       [ 5.7160575e-03, -2.3649796e-03,  2.0968895e-03, ...,
         5.3650416e-03,  7.2402111e-03,  1.4226218e-03],
       [-8.1616519e-03,  4.6124598e-03, -8.5368557e-03, ...,
         7.2550117e-03,  9.2172315e-03, -8.0767851e-03]], dtype=float32),
                    'w': array([[[ 0.0024759 , -0.00276686,  0.0023287 , ..., -0.00156066,
         -0.00649969,  0.00501549],
        [ 0.00389328,  0.00884204, -0.00255053, ..., -0.0137577 ,
         -0.01047342,  0.00884087],
        [-0.01649504,  0.00430972,  0.00274404, ...,  0.02594932,
         -0.04495201, -0.01399373],
        ...,
        [ 0.00888101, -0.00785696,  0.00229611, ...,  0.00703543,
         -0.007503  ,  0.00540859],
        [ 0.0085597 , -0.01634004,  0.00304436, ..., -0.00409839,
         -0.00071542,  0.00278309],
        [-0.00284988, -0.00032437, -0.01045181, ...,  0.01348813,
         -0.02370471,  0.00539708]],

[[ 0.00608244,  0.02770545, -0.02762934, ...,  0.03061771,
          0.01410207,  0.01474077],
        [ 0.00981219,  0.01534565,  0.0136242 , ..., -0.01917742,
         -0.00667843,  0.01681585],
        [-0.04855379,  0.00289034,  0.00202913, ...,  0.00555997,
         -0.05179906, -0.02609984],
        ...,
        [-0.00565242,  0.00732731,  0.01745838, ...,  0.01264094,
         -0.00761456,  0.00522908],
        [ 0.01639871, -0.02497045, -0.01113817, ...,  0.0061512 ,
          0.02803803,  0.00157584],
        [ 0.0116243 , -0.00251193, -0.02997128, ...,  0.0294893 ,
         -0.0202815 ,  0.01254657]],

[[ 0.01289525,  0.00492417, -0.00206911, ...,  0.00644436,
         -0.01303434, -0.00458589],
        [ 0.00940402,  0.01113505,  0.00721233, ..., -0.00587549,
          0.00453107,  0.01256718],
        [-0.03795905,  0.00221916,  0.00705938, ...,  0.01798605,
         -0.02529246, -0.01099175],
        ...,
        [ 0.00475134,  0.00634999,  0.00695073, ...,  0.00519345,
         -0.01645302, -0.0061229 ],
        [ 0.00143314, -0.00440471, -0.01998543, ..., -0.01487721,
          0.02314664, -0.00364762],
        [ 0.00416489, -0.00348652, -0.00756539, ..., -0.00339929,
         -0.00713988, -0.00030248]],

...,

[[ 0.00480618, -0.00772553,  0.00185248, ...,  0.00013539,
         -0.00468809, -0.00372351],
        [-0.00208754, -0.0050767 ,  0.00052303, ...,  0.01065702,
         -0.0014551 , -0.00957287],
        [-0.01182466,  0.00476738,  0.0044201 , ..., -0.00630753,
          0.00071952, -0.00046174],
        ...,
        [ 0.00470348,  0.00718361,  0.01261005, ..., -0.00914126,
          0.01383711, -0.00549724],
        [ 0.00308931, -0.00429104, -0.0019209 , ..., -0.01468712,
         -0.00493272,  0.00781817],
        [ 0.00291294, -0.00811114, -0.0074482 , ..., -0.00104166,
         -0.00094387,  0.00107866]],

[[ 0.02018718,  0.0030035 , -0.00131768, ...,  0.01029284,
         -0.01045149,  0.00331207],
        [ 0.00158666, -0.00323767,  0.00963177, ..., -0.01503262,
         -0.00236805, -0.00345408],
        [-0.0191923 ,  0.00814822,  0.0164628 , ..., -0.00031022,
          0.00559428,  0.00750944],
        ...,
        [-0.0052482 , -0.0110186 ,  0.01560836, ..., -0.01775388,
          0.0141189 , -0.01672785],
        [-0.00039182, -0.00405642,  0.00014005, ..., -0.00327382,
          0.01413307, -0.01277879],
        [-0.00571873,  0.00983705,  0.00172778, ...,  0.02574504,
         -0.00744921, -0.00387888]],

[[ 0.00473459, -0.00507944,  0.00289661, ...,  0.00038999,
         -0.0081128 , -0.00403884],
        [-0.00585744, -0.00309845,  0.00449292, ...,  0.00646797,
          0.00730119,  0.00418908],
        [ 0.01212458, -0.00163699, -0.00837895, ..., -0.00508494,
          0.00191359, -0.01044772],
        ...,
        [-0.00874585,  0.01692164,  0.00437238, ...,  0.01866874,
          0.00768705,  0.01460969],
        [-0.00084818, -0.00683727, -0.00774377, ..., -0.00031492,
         -0.0020496 ,  0.00169231],
        [ 0.00199953,  0.01445957,  0.00079529, ...,  0.00466131,
         -0.00959731,  0.00840501]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.30883983, 0.29701173, 0.28105626, ..., 0.32049316,
         0.25769606, 0.30365366],
        [0.27592438, 0.35078168, 0.3304479 , ..., 0.33389112,
         0.33066425, 0.32942727],
        [0.33974594, 0.30376896, 0.3010826 , ..., 0.30770063,
         0.3261373 , 0.3405804 ],
        ...,
        [0.3283833 , 0.34557635, 0.38340414, ..., 0.28545702,
         0.30639136, 0.36298352],
        [0.3343052 , 0.29297924, 0.3705194 , ..., 0.29628062,
         0.3201834 , 0.31475648],
        [0.32277626, 0.32514212, 0.35435086, ..., 0.33257237,
         0.3680805 , 0.33801526]]], dtype=float32)
                }
            }
        },
        'x_layers_10': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': array([-0.0067778 , -0.01533606, -0.00765442, ...,  0.0045179 ,
       -0.00582889, -0.01096233], dtype=float32)
                    },
                    'linear': {
                        'w': array([[-0.03108777,  0.01383287, -0.01179256, ...,  0.00302204,
        -0.01438387, -0.01885613],
       [ 0.00535768, -0.04698981, -0.03603473, ..., -0.01139537,
        -0.06621313,  0.03310591],
       [ 0.04872718, -0.0184944 , -0.04604091, ..., -0.04631934,
        -0.02415592,  0.05293693],
       ...,
       [-0.00610507, -0.02534259, -0.02141761, ..., -0.01412116,
         0.00220612, -0.03082991],
       [-0.06125214,  0.01565345,  0.03463744, ...,  0.00639873,
         0.05287894, -0.03431875],
       [-0.01022425,  0.05079507, -0.00049477, ...,  0.01767915,
        -0.00905383,  0.06914392]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.0983022, 1.1869484, 1.0642064, ..., 1.2279091, 1.3065565,
        1.1312699]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': array([ 0.00332026,  0.00228718,  0.00081935, ...,  0.00143483,
       -0.00280289,  0.0016605 ], dtype=float32)
                    },
                    'linear': {
                        'w': array([[-0.01038639, -0.00440558, -0.01641116, ..., -0.05271889,
         0.00743694, -0.00781685],
       [-0.06110938, -0.03921739, -0.03602229, ...,  0.00572572,
         0.00305964,  0.0342207 ],
       [-0.02868134, -0.01991718,  0.02981003, ...,  0.02833821,
        -0.03532288,  0.02274652],
       ...,
       [-0.04388466, -0.04275953, -0.06564004, ...,  0.03234898,
         0.05879903, -0.04443722],
       [ 0.06963926, -0.01727262,  0.04664012, ..., -0.03569457,
        -0.00283924, -0.00034458],
       [ 0.00845499, -0.02555194, -0.02030128, ...,  0.06764334,
         0.03920732,  0.06198909]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.1923957, 1.453415 , 1.3592325, ..., 1.3414761, 1.3559438,
        1.651706 ]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': array([-0.00652681,  0.00998919,  0.01091337, ..., -0.00884033,
       -0.01000154,  0.00719102], dtype=float32),
                    'scale': array([ 0.00018895, -0.0016679 , -0.00290126, ..., -0.00919969,
       -0.00058326,  0.00534971], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': array([1.1182874, 1.1435981, 1.1135991, ..., 1.1002538, 1.118429 ,
       1.1571561], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': array([[-0.00028102, -0.00059104, -0.00017435, ..., -0.00119082,
        -0.00010263, -0.00071618],
       [ 0.00225487, -0.00275545,  0.00204913, ..., -0.00047768,
         0.00016692,  0.00026425],
       [ 0.00023483,  0.00177506, -0.00131258, ...,  0.0014215 ,
        -0.0040562 ,  0.00237817],
       ...,
       [ 0.00106694,  0.0002561 ,  0.00144923, ...,  0.0013506 ,
         0.00120613, -0.00084387],
       [ 0.00105687, -0.00163147, -0.00046463, ...,  0.00094481,
         0.00178576,  0.00060445],
       [ 0.00122252, -0.00197313,  0.00088491, ...,  0.00058077,
         0.00029764, -0.00264683]], dtype=float32),
                    'w': array([[[-1.22559890e-02, -8.39809235e-03, -1.48427067e-02, ...,
          6.22703508e-03,  1.86309591e-02,  3.24096270e-02],
        [-2.01482568e-02,  8.23511835e-03, -3.59878410e-03, ...,
          5.64934453e-03,  8.32179096e-03,  1.41915437e-02],
        [-2.01054364e-02, -6.93156384e-03, -9.31501039e-04, ...,
         -4.10634056e-02,  6.05812436e-03,  1.22190723e-02],
        ...,
        [-4.57775360e-03,  1.78224165e-02, -1.38880024e-02, ...,
         -1.27791585e-02,  1.98074467e-02, -3.26664071e-03],
        [ 4.63145878e-03,  7.87381578e-05, -2.41322033e-02, ...,
         -1.23561732e-02,  6.57504192e-03,  6.37360057e-03],
        [ 4.68460284e-03,  4.65884060e-03, -1.86497867e-02, ...,
          3.24863405e-03,  1.32136820e-02,  5.25168935e-03]],

[[-9.33110993e-03,  2.16640662e-02,  1.45568941e-02, ...,
          2.85669211e-02,  2.10292288e-03,  2.34060623e-02],
        [ 1.74203468e-03,  6.03777182e-04,  9.83542949e-03, ...,
         -3.02254339e-03,  2.46134792e-02, -1.45584624e-02],
        [ 2.48686746e-02,  5.89962862e-03, -2.33576167e-02, ...,
         -2.28236821e-02,  4.75968607e-03,  3.50653799e-03],
        ...,
        [-1.78693212e-03, -1.40904635e-02,  1.15085309e-02, ...,
          5.33613726e-04, -1.37730991e-03, -1.78048816e-02],
        [-4.76122927e-03, -2.46134261e-03, -6.01499388e-03, ...,
          4.62375861e-03, -1.24234725e-02,  7.65619939e-03],
        [ 6.24199677e-03, -1.23126097e-02, -7.11983116e-03, ...,
         -1.19737433e-02,  1.70836709e-02,  1.27385380e-02]],

[[ 7.84550887e-03,  1.64878145e-02,  1.49704758e-02, ...,
         -9.02089011e-03, -1.44302985e-02, -3.40348110e-03],
        [ 4.17827722e-03, -1.87995359e-02, -2.49980614e-02, ...,
         -1.30755920e-02, -5.24970656e-03,  4.44735168e-03],
        [-1.73772697e-03,  2.21553333e-02, -8.72129481e-03, ...,
         -4.68143076e-03,  6.37933007e-03, -4.23845323e-03],
        ...,
        [ 1.10918321e-02, -1.17540890e-02,  1.29305832e-02, ...,
         -4.84592095e-03,  3.77263036e-03, -1.85808912e-03],
        [-9.93069168e-03,  8.13685730e-03,  3.15489732e-02, ...,
         -6.26456272e-03,  1.46055082e-02,  3.77353863e-03],
        [ 2.12038402e-02, -2.66305730e-03, -3.35844830e-02, ...,
          7.59634282e-03,  2.79865600e-02, -8.21714010e-03]],

...,

[[ 4.45938576e-03, -1.98688991e-02, -2.58294344e-02, ...,
          2.40076496e-03,  1.67861395e-02, -2.27218494e-03],
        [ 2.83571146e-02,  6.38406631e-03, -1.14099970e-02, ...,
         -6.16428396e-03, -5.20368386e-03, -1.55127551e-02],
        [-4.07850929e-02, -7.41915638e-03,  1.75307486e-02, ...,
         -8.14447552e-03,  2.47047432e-02, -2.11377116e-03],
        ...,
        [ 3.06497142e-02, -1.88986845e-02,  3.68725806e-02, ...,
         -1.07228439e-02, -1.59338117e-02,  1.52657116e-02],
        [ 8.17717146e-03, -2.32098773e-02, -2.22791266e-03, ...,
          1.75007638e-02,  7.45276688e-03,  2.72153248e-03],
        [ 8.80232640e-03, -1.40034547e-03,  7.92576466e-03, ...,
          2.64830943e-02,  2.43100859e-02,  1.96908843e-02]],

[[-3.02117150e-02, -1.86287034e-02, -1.25903729e-02, ...,
          1.77729372e-02, -3.20590800e-03, -2.27465411e-03],
        [ 1.22794071e-02, -5.00447862e-03, -1.01554077e-02, ...,
          8.89281183e-03, -2.84617804e-02, -1.21193705e-02],
        [-8.42523109e-03,  8.38035159e-03,  6.19815895e-03, ...,
          1.33166850e-05, -1.48215313e-02, -4.31424612e-03],
        ...,
        [-4.31096777e-02,  5.23513881e-03, -2.70542651e-02, ...,
         -1.62883531e-02,  1.30473943e-02, -4.56494391e-02],
        [ 8.53090547e-03, -2.16933284e-02, -2.11660322e-02, ...,
          1.69779006e-02, -9.84099461e-04,  5.07520558e-03],
        [-3.00326594e-03,  6.68262783e-03,  2.82763615e-02, ...,
         -1.81443505e-02,  1.39162512e-02, -1.42788538e-03]],

[[ 1.13208451e-04,  3.52317952e-02,  4.93009761e-03, ...,
          1.20255938e-02,  5.44467615e-03,  1.82285514e-02],
        [ 1.27479937e-02, -1.24411648e-02,  1.48131000e-02, ...,
         -2.89926236e-03,  4.92918771e-03,  9.24422313e-03],
        [ 2.80147996e-02,  2.45938282e-02, -4.92357509e-03, ...,
         -5.41405566e-03, -1.25533426e-02,  2.44517159e-02],
        ...,
        [ 2.46858466e-02, -2.21450329e-02, -1.72019340e-02, ...,
          2.06563505e-03,  1.24617934e-03,  1.31983301e-02],
        [ 6.06509508e-04, -6.05794648e-03,  2.30345670e-02, ...,
         -7.03589944e-03, -1.05391983e-02, -1.45047577e-02],
        [-7.19427271e-03, -2.63856980e-03,  6.21147733e-03, ...,
         -2.99184263e-04, -2.09438652e-02,  6.05939981e-03]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.5430286 , 0.55199766, 0.5347722 , ..., 0.5254196 ,
         0.65404433, 0.55363643],
        [0.50268894, 0.59888387, 0.56137985, ..., 0.55956846,
         0.56656533, 0.40336287],
        [0.5235853 , 0.56373084, 0.43853617, ..., 0.67811704,
         0.51538444, 0.4470662 ],
        ...,
        [0.5340978 , 0.49901468, 0.53142494, ..., 0.50598806,
         0.5341264 , 0.5080169 ],
        [0.4407545 , 0.5809068 , 0.4983939 , ..., 0.4916404 ,
         0.52818036, 0.3943283 ],
        [0.43778446, 0.5119181 , 0.58700705, ..., 0.50979286,
         0.5423889 , 0.602302  ]]], dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': array([0.2826307 , 0.29122394, 0.2804533 , 0.27769658, 0.2847127 ,
       0.24761622, 0.2710216 , 0.3152247 , 0.2765104 , 0.2484727 ,
       0.2533266 , 0.27915084, 0.24555965, 0.3073783 , 0.2655005 ,
       0.21189591, 0.35232496, 0.30432513, 0.30837816, 0.25320613,
       0.29300117, 0.25749105, 0.30691883, 0.2562598 , 0.31728888,
       0.26139805, 0.24298659, 0.29966286, 0.24946421, 0.28548405,
       0.21169291, 0.26257178, 0.31407136, 0.2906975 , 0.27783373,
       0.26280415, 0.2737751 , 0.30103633, 0.2973439 , 0.25732696,
       0.27588543, 0.2568364 , 0.3071738 , 0.24696194, 0.29834172,
       0.23746881, 0.27979898, 0.23967174, 0.25484917, 0.31370804,
       0.24649821, 0.2343228 , 0.24406944, 0.30672845, 0.29227123,
       0.24194668, 0.24848543, 0.27132946, 0.25255135, 0.24560043,
       0.28916624, 0.29397386, 0.2424356 , 0.27622077, 0.30238086,
       0.34377092, 0.2524194 , 0.26836666, 0.27820814, 0.24207515,
       0.27608994, 0.27313074, 0.29991195, 0.2935175 , 0.23672445,
       0.28691012, 0.24444716, 0.24724962, 0.2769544 , 0.20427427],
      dtype=float32)
                },
                'post': {
                    'b': array([ 7.9627399e-04,  6.5674139e-03,  5.6891823e-03, ...,
        4.3108783e-05, -3.6937576e-03,  2.3444982e-03], dtype=float32),
                    'w': array([[[ 0.00146442,  0.01960428, -0.00436989, ..., -0.0122232 ,
          0.00920983, -0.00245794],
        [-0.0095711 ,  0.00907172, -0.0116214 , ..., -0.00354761,
         -0.00579774,  0.00126938],
        [ 0.00276704,  0.00060504, -0.00706686, ..., -0.00294393,
          0.00962784,  0.01267469],
        ...,
        [ 0.00700635, -0.02775826,  0.00443999, ...,  0.01711492,
         -0.021339  ,  0.00733781],
        [-0.0081798 ,  0.01323767,  0.01253227, ..., -0.01629487,
          0.0106214 ,  0.02292852],
        [ 0.00385438,  0.0126046 ,  0.00792869, ...,  0.01123373,
         -0.00201993, -0.00607193]],

[[-0.01066706, -0.01185437,  0.00581993, ..., -0.00038126,
          0.00969159,  0.00856856],
        [ 0.01023707,  0.01450412, -0.00871626, ..., -0.01748885,
          0.00702912,  0.00037257],
        [-0.0103297 ,  0.02880418,  0.01049584, ..., -0.00153923,
         -0.00138126, -0.00856328],
        ...,
        [ 0.00176534, -0.01488073, -0.01772467, ..., -0.0253991 ,
          0.03928906,  0.02362941],
        [ 0.01271499,  0.00404142,  0.00724872, ..., -0.01562272,
         -0.01194645, -0.01826903],
        [-0.01120819,  0.00763887, -0.00247908, ..., -0.01165817,
          0.00719823,  0.00766752]],

[[ 0.0111996 ,  0.01693269, -0.00770767, ...,  0.00399856,
         -0.01616363, -0.00975752],
        [ 0.00795718,  0.00687884,  0.01115433, ..., -0.00098466,
         -0.00417739, -0.01695237],
        [-0.00133621, -0.02013697, -0.012535  , ..., -0.02007058,
         -0.0026116 ,  0.01423578],
        ...,
        [ 0.01130384,  0.0069552 ,  0.00096859, ..., -0.00324061,
          0.01272114,  0.00335166],
        [-0.00116659, -0.01251554,  0.00596234, ..., -0.00872863,
          0.01376314,  0.00547627],
        [-0.0114745 ,  0.01857201, -0.00061913, ...,  0.01724387,
         -0.00529594, -0.00799787]],

...,

[[-0.00822657, -0.01715343, -0.02105782, ...,  0.0093978 ,
          0.00354596, -0.00543864],
        [ 0.02705701,  0.00017979,  0.00241849, ..., -0.00759695,
         -0.00314767, -0.00162846],
        [-0.00841648,  0.03166628, -0.02671383, ..., -0.01387258,
         -0.00129089,  0.01384841],
        ...,
        [ 0.01240158,  0.00602521, -0.01073924, ...,  0.0005019 ,
          0.01391591,  0.00223366],
        [ 0.02888446, -0.00281212, -0.00594539, ...,  0.00623288,
          0.00572395, -0.00381602],
        [-0.00309395, -0.00522599,  0.01051544, ...,  0.01084724,
         -0.00763254, -0.02543153]],

[[-0.01109877, -0.00028809, -0.004021  , ..., -0.01404384,
         -0.01532735,  0.01193627],
        [-0.01674422, -0.0081592 ,  0.00955065, ...,  0.00220343,
         -0.00097354, -0.00783897],
        [ 0.00512082, -0.00585231, -0.00934302, ..., -0.00866921,
         -0.00591186, -0.01668595],
        ...,
        [ 0.01206996, -0.0002157 ,  0.01924735, ...,  0.01269434,
         -0.03069581, -0.01251041],
        [-0.00685677,  0.00429711, -0.0035953 , ..., -0.0210191 ,
          0.00736486,  0.00364637],
        [-0.00293386, -0.02583666, -0.00300108, ...,  0.00027271,
          0.01230213, -0.01810557]],

[[-0.00086033, -0.02251566,  0.0042542 , ..., -0.00318098,
          0.00196227, -0.00998912],
        [-0.00541877,  0.00864972, -0.00619451, ..., -0.02292626,
         -0.00532148,  0.01265658],
        [ 0.00966086,  0.00343994, -0.00895778, ...,  0.01743233,
          0.00556804,  0.03041199],
        ...,
        [-0.00209922, -0.00457141,  0.00087416, ...,  0.00012216,
         -0.00576033,  0.01509899],
        [ 0.00219567, -0.01285906,  0.00986449, ...,  0.00465276,
          0.00352433, -0.02275107],
        [-0.00331315,  0.00438775, -0.00717978, ..., -0.00826449,
         -0.00581287, -0.002987  ]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.42440635, 0.425131  , 0.41870847, ..., 0.4401614 ,
         0.45612475, 0.43353298],
        [0.40715417, 0.3904497 , 0.4005741 , ..., 0.38921246,
         0.3759506 , 0.43682155],
        [0.4335082 , 0.43643102, 0.4243714 , ..., 0.4318623 ,
         0.4170245 , 0.45153198],
        ...,
        [0.42948407, 0.4458929 , 0.4188972 , ..., 0.43029514,
         0.43349725, 0.42776662],
        [0.38637897, 0.39982352, 0.39171535, ..., 0.39805308,
         0.36692768, 0.40916786],
        [0.4356096 , 0.40518612, 0.42820302, ..., 0.43957356,
         0.42772785, 0.43686023]]], dtype=float32)
                },
                'query': {
                    'b': array([[ 0.01600958,  0.00398826,  0.00773052, ..., -0.01548967,
         0.00642794,  0.01298024],
       [ 0.00263645, -0.00473251, -0.00969235, ...,  0.00956154,
        -0.00312625, -0.02954184],
       [-0.00979827, -0.00529572,  0.0223179 , ...,  0.00354051,
         0.00967805, -0.00740001],
       ...,
       [ 0.00558155,  0.0006147 ,  0.00570557, ...,  0.01541192,
        -0.00349259, -0.00577878],
       [ 0.03326915,  0.02490069, -0.03056262, ...,  0.0211593 ,
         0.02024012,  0.03402796],
       [-0.02150407, -0.00563736,  0.01245746, ...,  0.00610306,
        -0.01175688, -0.00551602]], dtype=float32),
                    'w': array([[[ 1.40356151e-02,  5.94120426e-03, -2.29084790e-02, ...,
         -5.35522103e-02, -1.32354945e-02,  1.50852511e-02],
        [ 1.75219439e-02, -6.20160834e-04, -4.98878174e-02, ...,
          2.22302740e-03,  4.82971693e-04, -3.72398198e-02],
        [ 2.58184019e-02,  1.29804909e-02, -1.26741165e-02, ...,
          1.46833034e-02, -1.93886776e-02, -2.34460309e-02],
        ...,
        [-3.15314792e-02, -8.34475458e-02, -4.39461842e-02, ...,
         -2.68571340e-02,  1.01708723e-02, -3.54705937e-02],
        [ 8.31054058e-03,  3.87277617e-03, -2.14894675e-02, ...,
         -1.33718094e-02, -2.90948711e-02,  2.14557368e-02],
        [-1.25645357e-03,  2.59165093e-02,  2.21974850e-02, ...,
          2.54525691e-02, -1.30615011e-02, -5.37769049e-02]],

[[-6.64277060e-04, -8.18970054e-03,  1.30345523e-02, ...,
          1.09456871e-02,  2.46415287e-02, -1.91208906e-03],
        [-1.29217217e-02, -2.13812981e-02,  2.84682168e-03, ...,
         -8.92780907e-03,  3.27471265e-04,  2.38504037e-02],
        [ 2.19151918e-02, -7.11217895e-03, -3.47008370e-02, ...,
          1.12959463e-02, -3.27894278e-02,  4.24545631e-03],
        ...,
        [ 1.13170519e-02, -2.20208839e-02, -1.64228417e-02, ...,
         -3.95518355e-03, -2.59853806e-02,  2.08602026e-02],
        [-2.24960651e-02,  7.19420798e-03, -1.53441839e-02, ...,
          9.06456728e-03, -1.22019444e-02, -1.92768183e-02],
        [ 3.00556030e-02,  8.02684389e-03, -1.72765739e-02, ...,
          2.26810705e-02,  1.89628769e-02,  9.30336118e-03]],

[[-2.70457529e-02, -7.48995750e-04,  4.50855754e-02, ...,
         -4.99687390e-03, -3.19675654e-02,  8.66994727e-03],
        [-1.34260831e-02,  8.75523034e-03, -1.26483012e-03, ...,
         -9.85739566e-03,  1.18895387e-02, -9.07773967e-04],
        [ 4.81377682e-03, -6.72455924e-03, -1.63849071e-02, ...,
          8.82298220e-03, -3.05827297e-02,  5.66488244e-02],
        ...,
        [ 6.07986795e-03,  5.82352374e-03, -2.14950480e-02, ...,
         -1.47261210e-02, -1.25445360e-02,  3.85936089e-02],
        [ 7.05899438e-05, -3.04246377e-02,  8.01263656e-03, ...,
          3.89043777e-03,  1.24686677e-02,  2.35836934e-02],
        [ 8.84465780e-03,  2.22626906e-02, -3.98728484e-03, ...,
          2.41415557e-02, -3.57260043e-03,  9.29407962e-03]],

...,

[[ 2.08282378e-02,  1.70838758e-02,  1.15278456e-02, ...,
         -2.19811630e-02, -7.07403850e-03,  2.64738090e-02],
        [-1.13856290e-02,  1.46905966e-02, -2.22666245e-02, ...,
          6.35353755e-03, -5.01705892e-03, -6.70881150e-03],
        [-1.82765834e-02, -1.30835092e-02,  6.77556731e-03, ...,
          2.31810864e-02, -1.24425562e-02,  1.62019413e-02],
        ...,
        [ 1.67714581e-02, -1.40784169e-02,  1.67892873e-02, ...,
         -1.22007690e-02,  2.40733866e-02, -1.20365014e-02],
        [ 2.81955060e-02, -1.64980069e-02,  4.70353581e-04, ...,
          6.99397642e-03,  2.87616942e-02,  2.75215060e-02],
        [ 3.40689160e-02, -1.82653125e-02,  1.74584929e-02, ...,
          8.72420962e-04,  2.08970513e-02,  3.98537740e-02]],

[[ 1.92598868e-02,  1.36986021e-02, -1.62966251e-02, ...,
         -2.47459114e-02,  2.18021106e-02,  4.18713177e-03],
        [ 3.45486365e-02,  1.71644408e-02, -3.59831899e-02, ...,
         -1.08249802e-02, -2.64170277e-03, -8.20003729e-03],
        [ 2.71651405e-03, -5.85488917e-04,  2.76921317e-02, ...,
         -3.33124809e-02,  1.70815289e-02, -1.39574464e-02],
        ...,
        [-1.97923444e-02,  1.73337311e-02, -5.96687980e-02, ...,
         -2.29241466e-03,  8.03769007e-03,  6.48612753e-02],
        [ 1.03499424e-02,  3.08426302e-02, -2.72807442e-02, ...,
          2.93351635e-02,  1.75606087e-02, -1.07692147e-03],
        [-3.39239687e-02, -7.36508938e-03,  1.47305522e-02, ...,
          3.31201777e-02, -4.31992933e-02,  2.14105286e-02]],

[[-1.60638113e-02, -6.68555452e-03,  1.93832051e-02, ...,
         -3.06526269e-03, -1.15770791e-02,  3.39199603e-03],
        [ 1.71611663e-02,  2.29309052e-02,  4.75622788e-02, ...,
          6.47983840e-03, -8.60652328e-03,  1.79007335e-03],
        [ 4.43298332e-02,  4.19816077e-02, -2.22983640e-02, ...,
         -2.87520867e-02, -2.16403510e-02,  6.30730949e-03],
        ...,
        [-3.26161981e-02,  2.95277941e-03, -1.02727003e-02, ...,
          1.27500594e-02,  1.43683655e-02, -2.82541923e-02],
        [-6.02143863e-03, -4.94311610e-03, -2.88585536e-02, ...,
         -3.91545370e-02, -2.86819078e-02, -9.28332284e-03],
        [-9.95085947e-03,  1.78441207e-03,  2.01135557e-02, ...,
          1.29047967e-03,  7.24823726e-03, -7.70971971e-03]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.8421002 , 0.81193775, 0.89382696, ..., 0.93825006,
         0.7554174 , 0.8918245 ],
        [0.7301947 , 0.63516986, 0.8881664 , ..., 0.6471393 ,
         0.710433  , 0.77922976],
        [0.8946232 , 0.77992713, 0.8609932 , ..., 0.65715486,
         0.85435885, 0.8843978 ],
        ...,
        [0.9594674 , 0.87042147, 0.9013787 , ..., 0.78006744,
         0.8389445 , 0.99407095],
        [0.55474824, 0.6269323 , 0.6206412 , ..., 0.657968  ,
         0.6847625 , 0.5224853 ],
        [0.79883635, 0.68205214, 0.7338766 , ..., 0.7869946 ,
         0.71955234, 0.8200522 ]]], dtype=float32)
                },
                'value': {
                    'b': array([[ 4.5682947e-04,  1.1954156e-04, -3.4605192e-03, ...,
        -2.7432817e-03, -6.2608662e-05, -2.2155052e-04],
       [ 3.4376332e-03,  4.1965075e-05,  2.1032067e-05, ...,
         1.5058093e-03, -1.0985093e-03,  3.8899151e-03],
       [ 7.2707783e-04, -6.1844854e-04,  1.3926377e-03, ...,
         3.1390032e-03, -6.2617409e-04,  2.1017287e-03],
       ...,
       [ 1.3336381e-03, -2.3447198e-03,  1.5440906e-03, ...,
         9.3644871e-05,  2.7158058e-03, -3.3990832e-03],
       [ 3.1390661e-04, -3.3300222e-04,  6.3425634e-04, ...,
        -6.3325587e-04, -9.4877614e-04, -6.7050627e-04],
       [ 2.8241735e-03, -7.6000334e-04, -5.7119469e-04, ...,
        -1.2217826e-03, -2.1154580e-04,  6.1394530e-04]], dtype=float32),
                    'w': array([[[ 1.72701497e-02, -6.67018292e-04, -8.94597825e-03, ...,
         -3.28017096e-03, -4.27454263e-02,  6.90392451e-03],
        [-2.00283024e-02, -2.27912087e-02,  3.64550166e-02, ...,
          1.12843597e-02, -7.17920298e-03,  3.81224486e-03],
        [-1.83604751e-02, -5.59525704e-03,  1.36792324e-02, ...,
          1.07971877e-02,  1.05734896e-02,  5.53275226e-04],
        ...,
        [ 1.50924991e-03,  1.53016811e-02,  7.70701421e-03, ...,
          2.38751504e-03,  3.14462073e-02,  2.86122821e-02],
        [-1.59093644e-02,  1.95494387e-02, -4.26613865e-03, ...,
          4.92176507e-04,  1.68949869e-02, -1.51739400e-02],
        [ 6.87260740e-03, -5.25096944e-03,  9.77834454e-04, ...,
         -9.87444259e-03,  2.22718250e-03,  1.64193325e-02]],

[[-2.13359180e-03, -1.58356354e-02, -2.50484911e-03, ...,
         -4.92309977e-04, -2.19545402e-02,  2.24699965e-03],
        [-9.16270725e-03, -1.33322617e-02,  3.90400994e-03, ...,
          1.53583670e-02,  1.30426278e-02, -2.83010537e-03],
        [ 2.33015930e-03, -1.37863839e-02, -1.89013407e-02, ...,
          1.43465176e-02,  5.64457057e-03,  1.97897088e-02],
        ...,
        [-1.58038994e-04, -2.03826390e-02,  4.86420467e-03, ...,
          1.91079290e-03, -2.07062531e-03, -2.11698636e-02],
        [-9.69564728e-03,  3.96481901e-03, -1.16651980e-02, ...,
         -6.88072469e-04,  6.54182024e-03, -1.06122680e-02],
        [-1.14566712e-02, -1.26549331e-02, -6.56894548e-03, ...,
         -1.56719461e-02, -5.79242548e-03, -6.15533954e-03]],

[[ 1.43140869e-03, -9.23580583e-03,  2.47010402e-02, ...,
         -1.28509058e-03,  6.37983065e-03, -7.71866972e-03],
        [-2.36263946e-02, -6.01142226e-03,  1.22867851e-03, ...,
          1.05331279e-02,  1.07444506e-02,  9.33971256e-03],
        [ 8.64060316e-03,  1.79494005e-02,  2.08592732e-02, ...,
         -2.49500689e-03, -4.60716337e-03, -1.55074010e-02],
        ...,
        [ 7.10266829e-03,  2.40555853e-02, -1.93814200e-03, ...,
          1.73993614e-02, -7.08616851e-03, -7.16193765e-03],
        [ 1.10008465e-02,  7.42531754e-03, -3.49915237e-04, ...,
         -1.13664558e-02,  1.72439020e-03, -1.46749138e-03],
        [ 6.11985149e-03, -3.04801576e-03, -1.11195026e-02, ...,
          3.82037740e-03,  4.31295205e-03,  7.19272532e-03]],

...,

[[-1.27223963e-02, -5.47991926e-03,  1.55010270e-02, ...,
         -1.09664602e-02, -1.63220279e-02,  3.23878857e-03],
        [ 2.58695986e-03,  8.04579537e-03,  6.82268990e-03, ...,
          1.33885522e-04,  2.75731981e-02,  1.91100948e-02],
        [ 1.36258435e-02, -7.79159553e-03,  1.33837545e-02, ...,
          3.53813963e-03, -5.63319819e-03,  1.27605265e-02],
        ...,
        [ 5.44381468e-03,  7.57290889e-03,  1.40039977e-02, ...,
          1.52988266e-02, -2.29196902e-02, -1.93062145e-02],
        [-2.87235598e-03, -4.21551894e-03,  8.06136057e-04, ...,
         -7.09132804e-03,  7.71719357e-03,  1.12887966e-02],
        [-5.87580679e-03, -1.75770093e-02,  2.83001480e-03, ...,
         -2.22835466e-02,  5.83054731e-03,  1.95922442e-02]],

[[ 9.04059783e-03,  6.03276258e-03,  7.76248379e-03, ...,
          1.72925275e-02,  1.24517083e-02, -4.21914756e-02],
        [-1.35183465e-02, -1.85723603e-02,  6.26749964e-03, ...,
          1.12873241e-02, -2.35499516e-02,  1.93779217e-03],
        [-8.21761508e-03, -5.12261875e-03,  7.94678368e-03, ...,
          2.76229251e-02,  1.26284373e-03, -2.03299578e-02],
        ...,
        [-1.15115205e-02, -9.66045167e-03,  3.15833208e-03, ...,
          1.17430668e-02,  8.09106138e-03,  1.85281830e-03],
        [-9.72276204e-04, -1.57450642e-02, -8.09854083e-03, ...,
          1.94889144e-03,  1.30797792e-02, -1.27516566e-02],
        [-2.35087723e-02,  2.47020088e-02,  4.70920652e-03, ...,
          1.25914551e-02,  3.74155316e-05,  1.47644766e-02]],

[[ 6.84548286e-04,  2.69682538e-02, -1.54047934e-02, ...,
          1.53855504e-02,  8.06880649e-03,  4.89617931e-03],
        [-1.54312905e-02, -6.75051892e-03,  1.15704141e-03, ...,
          6.85672089e-03,  1.08955754e-02, -1.00796996e-02],
        [-1.09128449e-02, -1.59259364e-02,  2.11769976e-02, ...,
          1.58200483e-03,  1.07090431e-03, -1.44510455e-02],
        ...,
        [-1.52886715e-02,  1.66227240e-02,  7.04474747e-03, ...,
          2.01132130e-02,  4.22550086e-03,  6.48447312e-03],
        [ 4.47970722e-03,  5.69607876e-03,  1.09875211e-02, ...,
          3.46971527e-02,  5.85914729e-03,  2.69552274e-03],
        [-1.18758455e-02,  1.96109060e-04, -4.45304206e-03, ...,
         -1.07747018e-02,  6.75870990e-03,  9.27476212e-03]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.46890715, 0.4582537 , 0.47675732, ..., 0.4652386 ,
         0.48103163, 0.4889309 ],
        [0.47576198, 0.45147675, 0.45686805, ..., 0.45462874,
         0.4196397 , 0.4880855 ],
        [0.5036832 , 0.47039992, 0.45430622, ..., 0.48450834,
         0.45560613, 0.48061076],
        ...,
        [0.46243483, 0.48713416, 0.4748612 , ..., 0.4623541 ,
         0.48331   , 0.47656044],
        [0.4192446 , 0.45171773, 0.44531536, ..., 0.44544938,
         0.40857542, 0.46341497],
        [0.45750064, 0.4541959 , 0.47714874, ..., 0.48058105,
         0.46523094, 0.47489065]]], dtype=float32)
                }
            }
        },
        'x_layers_11': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': array([-0.00175118, -0.00260024, -0.07228219, ..., -0.00627134,
        0.00413736, -0.00720975], dtype=float32)
                    },
                    'linear': {
                        'w': array([[ 0.02213248, -0.00037442,  0.06455456, ..., -0.00868579,
         0.0104142 , -0.02710419],
       [ 0.03270084,  0.0241459 , -0.04848697, ..., -0.03151397,
         0.03347551,  0.00065035],
       [-0.00683347,  0.02788812, -0.01665919, ...,  0.0314375 ,
         0.02865025, -0.0341912 ],
       ...,
       [ 0.01531753,  0.02818437,  0.05320358, ...,  0.03597291,
        -0.00072661,  0.04358203],
       [ 0.02493588,  0.03074985,  0.03032211, ...,  0.00291781,
         0.00067245, -0.0406046 ],
       [-0.03414572, -0.01924999, -0.00704158, ..., -0.00375799,
         0.03458655,  0.01684766]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[0.9200644, 1.0077778, 2.0899606, ..., 1.173255 , 1.3468095,
        1.1368102]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': array([ 0.00636536,  0.00142258, -0.00197367, ...,  0.00075185,
       -0.00083699,  0.00319561], dtype=float32)
                    },
                    'linear': {
                        'w': array([[ 0.00576453,  0.01392252, -0.01244743, ...,  0.02634548,
        -0.0390147 ,  0.05118013],
       [-0.04683452, -0.02094531, -0.05402165, ...,  0.02874108,
         0.01810433,  0.01536882],
       [-0.08094323, -0.17057261,  0.06222852, ...,  0.00614128,
         0.06856798, -0.02509728],
       ...,
       [-0.02571372,  0.11391928, -0.05397959, ..., -0.00348219,
         0.01380937, -0.06948423],
       [ 0.03470301, -0.04579668, -0.00139861, ..., -0.05194421,
        -0.05121345,  0.08306472],
       [ 0.01261504, -0.02599149,  0.00764601, ...,  0.0212124 ,
        -0.00760618,  0.00859129]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.2713861, 1.5184889, 1.3088137, ..., 1.4292699, 1.3142276,
        1.6887296]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': array([-9.9775520e-05,  1.3316037e-02,  1.2384055e-02, ...,
       -9.1059729e-03, -2.5081664e-02, -1.0536548e-02], dtype=float32),
                    'scale': array([ 0.00314306,  0.00437152,  0.00764915, ...,  0.02828045,
        0.01542678, -0.02259208], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': array([1.156839 , 1.1383926, 1.1070566, ..., 1.1095787, 1.1333247,
       1.1252159], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': array([[ 6.2043295e-04, -1.3959286e-03,  2.3537264e-03, ...,
         2.6737602e-04,  4.6616062e-03,  3.2361986e-03],
       [-2.2848645e-04, -1.9312546e-03, -7.2261330e-04, ...,
         3.8449914e-04,  2.2252691e-03, -3.6747850e-04],
       [-4.6333583e-04,  7.0525962e-04,  9.5361407e-04, ...,
        -2.3124523e-03, -1.8506297e-03,  4.0299914e-04],
       ...,
       [-8.8660774e-05, -1.8478214e-04, -3.6815304e-04, ...,
        -1.2149196e-03, -1.4899155e-03,  7.4451091e-05],
       [ 2.6595125e-03,  3.3997109e-03, -3.8310296e-03, ...,
         3.4757920e-03,  1.5175877e-03, -5.4881223e-03],
       [-6.1853061e-05, -3.0119701e-03,  1.3912504e-03, ...,
         3.1533630e-03,  8.3954731e-04, -2.0716980e-03]], dtype=float32),
                    'w': array([[[-0.00457982,  0.0079292 ,  0.01626714, ..., -0.02177358,
         -0.00753616,  0.0111543 ],
        [ 0.00736282, -0.01522866, -0.03390208, ...,  0.01205219,
          0.00368838, -0.02313531],
        [-0.00222201,  0.01081776, -0.0121273 , ...,  0.00954761,
          0.00297511,  0.02277266],
        ...,
        [-0.01193842,  0.02725654, -0.01004553, ..., -0.00874436,
         -0.01074831, -0.00778359],
        [ 0.01702505, -0.001029  , -0.01283343, ..., -0.01121063,
         -0.00362994, -0.00895869],
        [ 0.01208098, -0.01486417,  0.00574359, ..., -0.00670334,
         -0.01481716,  0.0051486 ]],

[[-0.0201987 , -0.02244218,  0.02493559, ...,  0.01063863,
          0.03113253, -0.0261899 ],
        [-0.00463472, -0.01037783,  0.011751  , ...,  0.01357768,
          0.01531901,  0.00621958],
        [-0.00490143, -0.01929298, -0.01632177, ...,  0.0314243 ,
          0.00510404, -0.0034225 ],
        ...,
        [ 0.01981692, -0.0038297 , -0.00277125, ..., -0.00398426,
          0.03406405,  0.022159  ],
        [ 0.00342297,  0.01845531, -0.01772549, ...,  0.00227201,
          0.03036357, -0.02189172],
        [-0.02262254, -0.00382184, -0.01107072, ...,  0.00248565,
         -0.00477183,  0.00756076]],

[[-0.00420213,  0.00172405, -0.0044528 , ..., -0.02730114,
         -0.00515185, -0.00871744],
        [-0.00775294, -0.01775882, -0.01258255, ..., -0.00312622,
         -0.01279273, -0.01131352],
        [ 0.00104673, -0.03599226, -0.00617872, ...,  0.04842951,
          0.01640603,  0.02405769],
        ...,
        [-0.00655407,  0.0259339 , -0.01562678, ...,  0.00079027,
         -0.00186389, -0.01522103],
        [-0.01466924,  0.00304388, -0.00875085, ..., -0.0011246 ,
         -0.0013253 ,  0.0053609 ],
        [-0.00148604, -0.00426759, -0.00711484, ..., -0.01309764,
         -0.02418446, -0.00673955]],

...,

[[-0.00054077, -0.01441031, -0.01318337, ..., -0.0155674 ,
         -0.00143004, -0.01898889],
        [-0.00300201,  0.02281963,  0.00653376, ...,  0.02473463,
          0.00196774, -0.02863496],
        [ 0.00202696, -0.00318841,  0.00057678, ...,  0.01588762,
         -0.01944294, -0.00230589],
        ...,
        [-0.00530122, -0.00388486,  0.00325499, ...,  0.02573116,
         -0.00344646,  0.01486101],
        [ 0.00294014,  0.00262161, -0.01326954, ...,  0.01108626,
          0.00471233, -0.00940747],
        [ 0.02328938,  0.00450493,  0.01872869, ..., -0.02414412,
         -0.00409967, -0.00039935]],

[[-0.00628797,  0.01608655, -0.0074989 , ...,  0.01741837,
          0.01408413, -0.01595278],
        [-0.00051053,  0.00011168, -0.00280871, ...,  0.00103641,
          0.005772  ,  0.01036084],
        [-0.00201812,  0.01018989,  0.02685075, ..., -0.03151053,
         -0.00190222,  0.00539631],
        ...,
        [-0.00211097, -0.01229035,  0.0256342 , ..., -0.0180869 ,
          0.00816433,  0.00191485],
        [-0.00479057, -0.0123396 ,  0.00897989, ..., -0.01457492,
          0.0158231 ,  0.00058097],
        [ 0.02179698,  0.0065661 ,  0.00655453, ..., -0.00327311,
         -0.01190026,  0.00449327]],

[[ 0.01201486, -0.0106921 , -0.01455065, ...,  0.0024754 ,
          0.01651331, -0.00582526],
        [ 0.00676857, -0.0060736 , -0.00843585, ..., -0.00529255,
          0.03072165, -0.00851097],
        [-0.00565433,  0.03521806, -0.0024964 , ..., -0.0134829 ,
          0.00173979,  0.01382122],
        ...,
        [ 0.0012847 ,  0.01398057, -0.01192073, ..., -0.00321589,
          0.00887635,  0.00224419],
        [ 0.00562963, -0.02689695, -0.00206297, ..., -0.01880339,
          0.00894797, -0.0019288 ],
        [ 0.01913709, -0.01316302, -0.03764893, ..., -0.0134073 ,
          0.02239419, -0.05215774]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.65483826, 0.60560954, 0.51017874, ..., 0.60101396,
         0.569222  , 0.49389306],
        [0.4817168 , 0.52972126, 0.64530927, ..., 0.5448027 ,
         0.53890574, 0.5650888 ],
        [0.4625575 , 0.5766899 , 0.54169947, ..., 0.6030134 ,
         0.48712596, 0.54196143],
        ...,
        [0.51594955, 0.5405832 , 0.5726122 , ..., 0.5704955 ,
         0.55448073, 0.4863739 ],
        [0.5126217 , 0.6237082 , 0.44152677, ..., 0.47121787,
         0.540265  , 0.4658949 ],
        [0.53947437, 0.4972289 , 0.62285125, ..., 0.48443028,
         0.49092695, 0.5999909 ]]], dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': array([0.3156545 , 0.27060285, 0.25025174, 0.34601673, 0.30242926,
       0.26294008, 0.31233415, 0.3209547 , 0.2849913 , 0.3288459 ,
       0.2951874 , 0.28546813, 0.34476817, 0.33740777, 0.284232  ,
       0.28655267, 0.26494822, 0.29337803, 0.30076003, 0.2882022 ,
       0.30189607, 0.271989  , 0.3013342 , 0.30964464, 0.26838246,
       0.2603131 , 0.32541406, 0.32179737, 0.35538927, 0.273313  ,
       0.3013977 , 0.2551599 , 0.29802316, 0.297576  , 0.27485722,
       0.3644293 , 0.29362065, 0.27971974, 0.32013175, 0.3102587 ,
       0.29194638, 0.26629296, 0.32570788, 0.32607472, 0.2795367 ,
       0.3140577 , 0.2892701 , 0.31884027, 0.30475014, 0.26978436,
       0.28342175, 0.31857198, 0.28328526, 0.29521295, 0.3464261 ,
       0.30882302, 0.29280803, 0.28246257, 0.28882617, 0.3388947 ,
       0.25662103, 0.3308743 , 0.32114318, 0.29870152, 0.30939123,
       0.28852645, 0.30617124, 0.2908906 , 0.30798033, 0.25875768,
       0.25241807, 0.33816648, 0.3175192 , 0.2886452 , 0.30905277,
       0.32988924, 0.3363946 , 0.30680224, 0.34127322, 0.32526475],
      dtype=float32)
                },
                'post': {
                    'b': array([ 5.6089791e-03,  8.0252551e-03,  5.5846493e-03, ...,
       -1.1894177e-03, -7.2966306e-03, -2.8434333e-05], dtype=float32),
                    'w': array([[[ 0.00616197, -0.01140843, -0.01220697, ...,  0.03211016,
         -0.00915002, -0.00320862],
        [-0.00163407, -0.00726635,  0.0026372 , ...,  0.01594584,
         -0.01256238, -0.00934431],
        [ 0.00269712, -0.0087405 , -0.00707724, ..., -0.01187651,
         -0.00985631,  0.01202096],
        ...,
        [-0.00177817, -0.0155329 ,  0.00039959, ..., -0.00509618,
          0.01596226, -0.0090281 ],
        [-0.00362233, -0.00166358,  0.010353  , ...,  0.01115924,
         -0.00344578, -0.01445935],
        [-0.00719647, -0.00679773,  0.00159126, ...,  0.00904022,
          0.00913885, -0.01345026]],

[[-0.02101772,  0.0023271 , -0.02241039, ...,  0.00969463,
         -0.0248913 ,  0.00367695],
        [-0.00095344, -0.01085228,  0.00195198, ..., -0.00032463,
         -0.00086249, -0.01440655],
        [ 0.00260988,  0.01833955, -0.01685798, ..., -0.00150539,
         -0.00219077, -0.00536269],
        ...,
        [-0.01157745,  0.00387631,  0.00098956, ..., -0.0190237 ,
         -0.00451367, -0.00499348],
        [-0.00846865, -0.00884161, -0.01536309, ..., -0.00709762,
          0.00124958,  0.01454337],
        [ 0.00352907,  0.00118892,  0.00079856, ...,  0.00428836,
         -0.00677275, -0.00062083]],

[[ 0.00527346, -0.01570467,  0.00518924, ...,  0.00133043,
          0.00971596, -0.01084871],
        [ 0.00400364, -0.00387563,  0.02051506, ..., -0.01235429,
          0.0043463 ,  0.01097707],
        [ 0.00741512,  0.0041224 , -0.00526709, ...,  0.02532259,
          0.00662523, -0.00377994],
        ...,
        [ 0.01415565, -0.00426505,  0.03681122, ...,  0.01064636,
         -0.00270368, -0.00990983],
        [-0.0034513 , -0.00753658, -0.013031  , ...,  0.03087043,
          0.00539378, -0.00348858],
        [-0.01530147, -0.01710001,  0.00850741, ...,  0.016956  ,
          0.01667179,  0.01122256]],

...,

[[-0.00915206, -0.00107384, -0.01732666, ...,  0.00080908,
         -0.01581459, -0.02295182],
        [ 0.00708712, -0.00128291, -0.0116279 , ..., -0.00595906,
          0.0249085 ,  0.01292838],
        [-0.01711876, -0.0257243 , -0.00384645, ...,  0.0013455 ,
         -0.00144086,  0.00735895],
        ...,
        [-0.01796792,  0.00160943, -0.0090777 , ..., -0.01585807,
         -0.0102811 ,  0.00546305],
        [ 0.00014763, -0.01729802, -0.00781043, ..., -0.01121969,
          0.01915416,  0.00109311],
        [ 0.01216643, -0.01559139,  0.00925548, ...,  0.01429955,
         -0.03162036, -0.0093506 ]],

[[-0.00386942,  0.00937686,  0.0146661 , ..., -0.0211518 ,
         -0.00266157, -0.00175802],
        [ 0.00132035, -0.01177001, -0.00124668, ..., -0.00543772,
         -0.00189881, -0.00991689],
        [-0.00866668,  0.00529445, -0.01004148, ...,  0.00385911,
          0.02554635,  0.0010934 ],
        ...,
        [-0.00413021, -0.00867042,  0.00941492, ..., -0.00267659,
          0.03558725,  0.00737279],
        [ 0.00622975,  0.00546824,  0.00530226, ...,  0.00883166,
          0.00025435,  0.001364  ],
        [ 0.00456269, -0.00797678, -0.01061194, ..., -0.00624754,
         -0.00368289,  0.00672814]],

[[ 0.00819326, -0.00378347, -0.00536378, ..., -0.00384975,
          0.00924117,  0.00283117],
        [ 0.00150048, -0.0014319 , -0.00483689, ...,  0.01057441,
          0.01947576,  0.00472962],
        [-0.0212515 ,  0.01883786, -0.00505871, ...,  0.0143664 ,
          0.00042881,  0.01159056],
        ...,
        [ 0.0004472 ,  0.01151392,  0.0013491 , ...,  0.00855052,
         -0.00520287, -0.01697323],
        [ 0.01939233, -0.00611647,  0.01255679, ...,  0.02077544,
         -0.01566506,  0.00383031],
        [-0.00312147, -0.01002034, -0.00377799, ...,  0.00377198,
         -0.00486637,  0.00745471]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.43909064, 0.46731344, 0.44162056, ..., 0.45429292,
         0.43974757, 0.46241692],
        [0.39665744, 0.42409852, 0.41744167, ..., 0.42897934,
         0.4505347 , 0.42544284],
        [0.45337102, 0.43128085, 0.397691  , ..., 0.43023804,
         0.43960187, 0.42477062],
        ...,
        [0.43815613, 0.40769398, 0.4149297 , ..., 0.4309527 ,
         0.44600788, 0.44566175],
        [0.44406974, 0.4677779 , 0.44883275, ..., 0.45317692,
         0.4471131 , 0.43732676],
        [0.45970654, 0.4399302 , 0.4069499 , ..., 0.42205188,
         0.4400244 , 0.4336801 ]]], dtype=float32)
                },
                'query': {
                    'b': array([[-0.01902875, -0.00392321, -0.01692069, ...,  0.01277379,
        -0.02017537, -0.01966368],
       [-0.00514293,  0.01013522,  0.00640971, ...,  0.00700698,
        -0.00208709,  0.0016113 ],
       [-0.01669493,  0.00687944,  0.00167014, ...,  0.00559093,
        -0.01526949, -0.01213987],
       ...,
       [ 0.01259618, -0.017484  ,  0.00524346, ...,  0.00086777,
         0.01313006,  0.01849369],
       [ 0.00576792,  0.00946417,  0.00764743, ..., -0.01395615,
        -0.00582852,  0.00378093],
       [-0.00593368,  0.00607188, -0.002292  , ..., -0.01927363,
        -0.00423207,  0.00327838]], dtype=float32),
                    'w': array([[[ 6.7107915e-03,  5.2108735e-02, -3.5726141e-02, ...,
          1.0018836e-02, -6.6985086e-02, -2.9997937e-02],
        [-4.8798919e-02, -3.0955179e-02, -4.5889658e-03, ...,
         -1.4168783e-02, -1.9515565e-02, -4.5938934e-03],
        [ 1.7948883e-02, -1.9678289e-02, -2.7322422e-03, ...,
          3.4518955e-05,  1.0276827e-02,  3.0288305e-02],
        ...,
        [ 3.2520901e-02,  3.1548835e-02,  3.2453798e-02, ...,
          3.0249842e-02, -1.3171470e-02, -3.0540736e-02],
        [-2.0047123e-02,  2.0765377e-02,  1.7037734e-02, ...,
         -3.1289443e-02,  3.2366186e-02,  2.4516651e-04],
        [-1.4139613e-03, -2.7877068e-02,  2.9665206e-03, ...,
         -8.7918406e-03, -3.1671613e-03, -6.0802251e-03]],

[[-1.5398514e-02,  9.0517215e-03, -4.1175298e-02, ...,
          2.5247632e-02, -2.5740143e-02, -1.3197754e-02],
        [ 2.1167113e-03, -2.7516490e-04, -2.0794403e-02, ...,
          8.8451179e-03, -3.5373135e-03,  1.0863101e-02],
        [-6.0629364e-02, -1.7886644e-02, -1.1412767e-03, ...,
          2.3840299e-02, -6.8457979e-03, -1.5402031e-02],
        ...,
        [ 4.6485676e-03, -1.6119034e-03, -6.2147551e-03, ...,
         -5.3045742e-02,  1.3163937e-02,  3.7071906e-02],
        [-1.7242387e-03,  3.4804703e-03, -2.6491882e-02, ...,
         -3.2698337e-02,  2.7583178e-02, -4.0955259e-03],
        [-3.3249643e-02,  8.4108170e-03,  1.7526602e-02, ...,
         -7.4118846e-03, -3.4503412e-02,  3.3055320e-02]],

[[-3.1678185e-02, -6.3735591e-03, -1.2943481e-02, ...,
          5.8178548e-03, -2.5701087e-02, -1.7416035e-03],
        [-6.1046705e-03, -2.0500453e-02, -3.1887960e-02, ...,
         -2.6464872e-03, -9.8156100e-03, -8.8180955e-03],
        [-1.3065789e-02,  1.5006047e-02, -1.3038984e-02, ...,
          1.0436822e-03, -5.2256063e-02,  3.3215946e-03],
        ...,
        [ 5.8997534e-02, -1.7044056e-02,  6.9897954e-04, ...,
         -1.5970137e-02,  1.7313166e-02, -4.2916336e-03],
        [-2.2724194e-03, -5.6460507e-02, -1.9561896e-02, ...,
          4.5353263e-03,  2.2484967e-02,  5.2606340e-02],
        [-3.9620679e-03, -1.4403533e-03, -1.9877153e-03, ...,
          4.0178779e-03, -4.3022860e-02,  2.6167121e-02]],

...,

[[ 7.4255988e-03,  6.5498860e-03,  8.8123837e-03, ...,
         -1.7822241e-02,  3.9919406e-02, -2.4898684e-02],
        [ 2.1847582e-03,  1.4291638e-02,  2.4705031e-03, ...,
          2.0050216e-02, -2.7914455e-02,  3.0144652e-02],
        [-3.1395327e-02,  1.7955898e-03, -1.2449417e-02, ...,
          2.7098171e-02, -1.7701644e-02, -2.9227233e-02],
        ...,
        [-3.1045333e-02,  1.4595293e-02,  2.9183826e-03, ...,
          6.5116766e-03, -5.3895572e-03,  2.6070766e-02],
        [-8.8878078e-03, -1.7661255e-02,  7.2191261e-02, ...,
         -9.6265236e-03, -6.4358683e-03,  6.7828201e-02],
        [-3.1275887e-02,  3.9514571e-02, -2.9801937e-02, ...,
         -2.8887761e-04, -8.4895547e-03, -6.9911694e-03]],

[[-8.9831362e-03, -1.8024940e-02,  3.7767717e-03, ...,
          1.4715036e-02, -1.7646234e-02, -4.1649252e-02],
        [-2.2571987e-02, -1.3751796e-02, -2.3236372e-02, ...,
         -2.4391491e-02,  2.2555089e-02, -1.1218257e-02],
        [ 6.0622431e-03, -1.5221324e-02, -7.2389385e-03, ...,
          2.6057046e-03,  9.2323516e-03,  2.0780820e-02],
        ...,
        [-3.1622655e-03,  3.9309144e-02,  3.4381237e-02, ...,
          3.5884291e-02, -4.2738756e-03,  2.1480158e-02],
        [ 1.3186349e-02, -4.2296544e-02, -1.8017067e-02, ...,
         -3.0487930e-02, -2.6397418e-02,  4.0354874e-02],
        [ 4.3010555e-04, -1.5165913e-02, -1.2764063e-02, ...,
         -3.8097229e-02,  1.3647714e-02, -8.6329188e-03]],

[[ 3.9989837e-02, -8.2005756e-03,  1.7767716e-02, ...,
          1.8468142e-03,  2.7596813e-03,  1.0810228e-02],
        [-3.4049381e-02,  1.8112855e-02,  4.7151122e-02, ...,
          2.6763078e-02, -3.0164555e-02, -1.5285182e-02],
        [ 2.9293558e-02,  1.1467175e-02, -1.0890862e-02, ...,
         -2.2795664e-03,  2.0938506e-02,  1.5154293e-02],
        ...,
        [-5.0077680e-02,  2.2626992e-02, -8.6043058e-03, ...,
          2.3196422e-02, -2.4148957e-03, -1.6109640e-02],
        [-5.3558229e-03, -7.1678106e-03, -9.8695131e-03, ...,
         -1.5392502e-02,  1.8007027e-03, -2.0988617e-02],
        [-1.3041359e-02, -4.1281167e-02, -1.1608910e-02, ...,
         -1.1827341e-02, -7.6007438e-03, -2.9945277e-02]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.76727873, 0.7167689 , 0.77897745, ..., 0.7998273 ,
         0.8349155 , 0.8138717 ],
        [0.7626718 , 0.6898988 , 0.634226  , ..., 0.75668496,
         0.7934055 , 0.71341103],
        [0.8279331 , 0.6869785 , 0.6840504 , ..., 0.60321784,
         0.86075515, 0.78291637],
        ...,
        [0.77018964, 0.8687415 , 0.681544  , ..., 0.767671  ,
         0.80532116, 0.72938377],
        [0.7489436 , 0.9255687 , 0.81932867, ..., 0.9513566 ,
         0.97494006, 0.92335695],
        [0.6699951 , 0.6605833 , 0.5263868 , ..., 0.8046062 ,
         0.6880497 , 0.74883664]]], dtype=float32)
                },
                'value': {
                    'b': array([[ 2.1569493e-04,  1.6116437e-03, -9.6050935e-04, ...,
        -6.5116951e-04, -6.0316140e-04,  4.0845378e-04],
       [ 2.7031007e-03,  2.5382696e-03,  2.3867956e-03, ...,
         3.8557494e-04,  1.9661232e-04,  1.8881332e-03],
       [ 7.4628938e-04,  2.0475746e-03, -1.8672572e-03, ...,
        -9.9990261e-04,  1.9131867e-03, -4.5626456e-04],
       ...,
       [-7.5348897e-04, -1.4569069e-03, -1.7060187e-04, ...,
        -2.4431029e-03, -4.1715769e-04,  4.0554241e-03],
       [ 2.6874433e-03,  2.4147295e-05, -3.1609035e-03, ...,
        -2.1084265e-03, -1.5388979e-03, -1.8308936e-03],
       [ 2.6015663e-03, -2.3570405e-05, -1.0428695e-03, ...,
        -1.4327377e-03,  3.6195872e-04,  5.5156567e-04]], dtype=float32),
                    'w': array([[[ 6.31503412e-04,  1.45085622e-03,  3.17231473e-03, ...,
         -3.25035416e-02,  2.26579071e-03, -1.32843880e-02],
        [-1.35540571e-02, -1.90303975e-03,  2.11610203e-03, ...,
         -2.68081836e-02,  1.45731904e-02,  2.98355296e-02],
        [-2.29272507e-02, -6.38898881e-03, -5.79964509e-03, ...,
          5.89815527e-03, -1.21480143e-02, -5.48514305e-04],
        ...,
        [ 2.93679964e-02,  1.65273417e-02, -7.91169144e-03, ...,
          3.39419208e-03,  5.04504656e-03,  4.24112082e-02],
        [ 5.78944013e-03,  4.27541137e-03, -2.87316274e-03, ...,
          7.80965155e-03, -1.54245207e-02,  3.92249711e-02],
        [ 9.51418560e-03, -6.88549364e-03,  1.10508539e-02, ...,
         -2.77678296e-02,  1.95166506e-02,  8.34298786e-03]],

[[ 7.28321684e-05,  1.08888466e-02,  3.29191200e-02, ...,
          4.54041176e-03,  2.65706517e-02, -4.64265421e-03],
        [ 6.72562094e-03, -1.68856829e-02,  2.15713251e-02, ...,
         -9.24612302e-03,  1.23386886e-02,  1.46161709e-02],
        [-1.86967999e-02, -2.36552469e-02,  1.11249238e-02, ...,
         -2.44511180e-02, -6.72320276e-03, -6.66756695e-03],
        ...,
        [ 1.38984632e-03, -1.34446111e-03, -8.61905050e-03, ...,
          1.83957480e-02,  8.14372208e-03,  7.35274795e-03],
        [ 4.99473326e-03,  7.30435736e-03,  3.96879902e-03, ...,
         -2.82498007e-03, -3.60422768e-02,  2.79442500e-03],
        [-8.63118935e-03,  1.90878951e-03, -2.41009775e-03, ...,
          5.82613703e-03,  8.67050607e-03, -3.18469224e-03]],

[[ 2.67320070e-02,  7.97787681e-03,  5.66311693e-03, ...,
         -8.75761826e-03, -1.52958669e-02, -2.23449338e-03],
        [-9.53639206e-03,  1.77496169e-02, -1.20923184e-02, ...,
         -7.46573613e-04,  1.69239212e-02,  7.67719839e-03],
        [-1.34969857e-02,  9.53905936e-03, -8.38374777e-04, ...,
         -7.19371345e-03,  2.54875757e-02, -9.41398088e-03],
        ...,
        [-6.10679854e-03,  1.15340967e-02, -2.58684647e-03, ...,
         -8.24218988e-03, -1.34088611e-02, -7.89085589e-03],
        [-2.41291393e-02, -7.42621906e-03, -2.79000308e-03, ...,
          1.46326004e-03, -8.06695037e-03, -1.68116614e-02],
        [ 1.13478582e-02,  1.84771549e-02,  2.46256702e-02, ...,
         -2.58951768e-04, -8.52987077e-03,  2.89389095e-03]],

...,

[[-4.72362386e-03,  2.18685791e-02,  2.06080228e-02, ...,
          2.27885321e-02,  5.52208396e-03,  2.80307420e-03],
        [ 6.93814736e-03, -1.64450370e-02, -9.28272435e-04, ...,
         -2.22967588e-03,  2.34239385e-03, -3.71391736e-02],
        [ 1.68014616e-02,  8.98404571e-04, -4.19066940e-03, ...,
         -3.64637119e-04, -2.06162371e-02, -2.11825185e-02],
        ...,
        [ 1.05074001e-02,  1.46402838e-03,  6.31070649e-03, ...,
         -6.07077358e-03, -1.54091360e-03, -6.06469205e-03],
        [-1.03934659e-02, -7.87057914e-03, -8.64219503e-04, ...,
          2.60232278e-04, -8.15128349e-03,  1.45747617e-03],
        [-8.99098255e-03, -9.93310474e-03,  2.38402933e-02, ...,
         -2.21021455e-02,  1.98403616e-02,  2.87384000e-02]],

[[-1.15994820e-02, -9.33563150e-03, -1.90857018e-03, ...,
          2.66199503e-02,  2.84725837e-02,  1.04084949e-03],
        [-1.06924130e-02,  7.14204228e-03,  1.98796950e-02, ...,
         -1.52518945e-02, -6.87469915e-03,  1.61008555e-02],
        [-2.07183622e-02,  7.97561370e-03,  1.37719316e-02, ...,
          1.03883063e-02, -2.63224375e-02, -2.98514287e-03],
        ...,
        [ 7.80780101e-03,  8.07005854e-04,  4.54832194e-03, ...,
         -1.08329821e-02, -2.70062741e-02, -1.40083767e-02],
        [-3.27079147e-02, -1.08146835e-02, -8.99208710e-03, ...,
         -9.17939842e-03,  2.96361372e-03,  9.23573412e-03],
        [-2.01473688e-03,  1.98760796e-02, -2.12014467e-03, ...,
          1.23345954e-02,  2.13192124e-03,  8.91683903e-03]],

[[-2.66917534e-02, -2.11770367e-02,  1.28460834e-02, ...,
          2.05245968e-02,  8.59414972e-03, -1.65698007e-02],
        [-4.99683758e-03,  4.65656910e-03,  1.11284396e-02, ...,
          1.69438235e-02,  2.64371349e-03, -1.35477455e-02],
        [ 1.36140212e-02, -6.97145518e-03, -1.70006659e-02, ...,
          1.33774430e-02, -4.16422449e-03, -4.53103613e-03],
        ...,
        [ 1.50281899e-02,  2.04638466e-02, -5.65202953e-03, ...,
         -2.92194309e-04,  1.09987482e-02,  2.95356428e-03],
        [ 3.92625947e-03, -1.02107776e-02, -3.78664173e-02, ...,
          4.30914294e-03,  9.04566702e-03,  1.14939539e-02],
        [ 2.49127694e-03,  1.19736474e-02,  8.10928084e-03, ...,
         -1.89916193e-02,  1.08265905e-02,  1.38882212e-02]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.47742516, 0.4799024 , 0.48178026, ..., 0.50168073,
         0.48671067, 0.49803993],
        [0.4379384 , 0.46320772, 0.46543244, ..., 0.47172067,
         0.4779946 , 0.47576147],
        [0.50264   , 0.4695091 , 0.44017136, ..., 0.4769782 ,
         0.4874883 , 0.4743337 ],
        ...,
        [0.4848537 , 0.48102137, 0.4770284 , ..., 0.4666153 ,
         0.47441497, 0.50505424],
        [0.49574783, 0.51803076, 0.48607302, ..., 0.47517514,
         0.48451996, 0.48085442],
        [0.49370846, 0.4892971 , 0.4521366 , ..., 0.46852696,
         0.48436654, 0.50114805]]], dtype=float32)
                }
            }
        },
        'x_layers_12': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': array([-0.005293  , -0.00186352, -0.00976755, ..., -0.00339723,
       -0.00447999,  0.00120996], dtype=float32)
                    },
                    'linear': {
                        'w': array([[-0.05177403,  0.03388924,  0.000139  , ...,  0.05519148,
         0.00972583, -0.02497132],
       [ 0.00069568,  0.0167134 , -0.02542666, ..., -0.063407  ,
         0.02492764, -0.00733338],
       [ 0.03444348,  0.00616424, -0.05731964, ..., -0.05109103,
        -0.04094215, -0.01131166],
       ...,
       [-0.02862806, -0.01231155, -0.01399679, ...,  0.05987118,
        -0.02321418,  0.04145474],
       [-0.00463293, -0.02615651,  0.01964143, ...,  0.06114722,
        -0.00820913,  0.02821478],
       [-0.0261556 ,  0.00931396, -0.02824642, ...,  0.03728772,
        -0.07729811, -0.00626508]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.0571388, 1.0269667, 1.03527  , ..., 1.2856728, 1.043925 ,
        0.929643 ]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': array([ 0.00122742,  0.00010827,  0.00017512, ..., -0.00082751,
       -0.00092059,  0.00197941], dtype=float32)
                    },
                    'linear': {
                        'w': array([[ 0.02870242,  0.01056961, -0.05915664, ...,  0.04323499,
        -0.00249853, -0.02161617],
       [ 0.01951083,  0.00519961,  0.00644839, ...,  0.03814481,
         0.01063481, -0.02081017],
       [ 0.01685014,  0.01569146, -0.01697563, ...,  0.01234141,
        -0.03345832,  0.04551524],
       ...,
       [ 0.04950652,  0.01376844, -0.00967372, ...,  0.06082512,
         0.09064892,  0.03400097],
       [-0.00100831, -0.02080236,  0.03347975, ...,  0.04448443,
         0.0319938 ,  0.01907505],
       [ 0.03527055, -0.01972844,  0.01550603, ...,  0.01602129,
         0.00324922, -0.01287165]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.2721705, 1.3986483, 1.273218 , ..., 1.3964808, 1.2481246,
        1.6003795]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': array([-0.00601908,  0.02212059,  0.02046599, ..., -0.00412445,
       -0.01955507, -0.00602043], dtype=float32),
                    'scale': array([0.0143517 , 0.00214911, 0.01108167, ..., 0.01265304, 0.03338145,
       0.0163041 ], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': array([1.1374506, 1.1442024, 1.0783095, ..., 1.1171334, 1.1079882,
       1.1521581], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': array([[-1.8685563e-04, -5.9532432e-04, -2.4054792e-04, ...,
         6.2968215e-04,  5.7730218e-04, -4.4597613e-04],
       [-2.2711651e-03,  2.0482612e-03, -2.2182227e-03, ...,
        -5.3416757e-04, -1.0647925e-03, -1.2073171e-03],
       [ 4.0595946e-03,  2.3135310e-03, -2.3099906e-03, ...,
         1.4927713e-03,  1.5450851e-03, -1.6390900e-04],
       ...,
       [ 2.7009632e-04,  5.9131434e-04,  1.3146417e-03, ...,
         8.6348550e-04,  1.2941496e-04,  1.5455941e-04],
       [-7.8688434e-04,  2.6643097e-03,  2.3450158e-04, ...,
         1.3286179e-03,  3.6279860e-04,  1.4461885e-03],
       [-6.0682447e-04,  1.9236694e-03,  2.6605760e-03, ...,
        -2.7408556e-03,  3.2813117e-05,  1.4253191e-03]], dtype=float32),
                    'w': array([[[-0.04404075,  0.0221399 ,  0.00467264, ...,  0.0320016 ,
          0.00014809, -0.00875474],
        [ 0.00923844, -0.0026586 ,  0.02337929, ..., -0.00236505,
          0.0094295 ,  0.00125714],
        [-0.04657682, -0.02400855, -0.01212377, ...,  0.01054189,
         -0.05642375, -0.00059953],
        ...,
        [ 0.03438158,  0.00055848, -0.00796749, ...,  0.02653286,
         -0.0355892 ,  0.01229315],
        [ 0.00275324,  0.0128751 , -0.02055732, ..., -0.00229726,
         -0.00360587,  0.0258247 ],
        [-0.01309112,  0.02698477,  0.04702034, ..., -0.00881093,
         -0.01423186, -0.04045918]],

[[ 0.00784455, -0.00358285, -0.00208098, ..., -0.00284979,
         -0.00200059, -0.03143811],
        [-0.00555067, -0.03134589,  0.01383472, ...,  0.02087029,
          0.01625624,  0.02094454],
        [ 0.03718068, -0.00865951, -0.0114981 , ...,  0.00183722,
         -0.00082921, -0.01832174],
        ...,
        [-0.00082864,  0.00861388, -0.01841935, ..., -0.03209327,
         -0.02881209,  0.02302638],
        [-0.01030933, -0.03367737,  0.0370855 , ..., -0.02565412,
          0.00903861, -0.01354067],
        [-0.01713261,  0.00071858, -0.01105236, ...,  0.01854027,
         -0.00121646, -0.0163816 ]],

[[ 0.00529315, -0.01076185,  0.00479513, ..., -0.00640778,
          0.00227136, -0.02275701],
        [ 0.01551528,  0.01069868,  0.00286569, ...,  0.0255737 ,
          0.02083129, -0.00256765],
        [-0.00903072, -0.00801174, -0.00946578, ..., -0.00016975,
         -0.03386201,  0.03029808],
        ...,
        [-0.03099167,  0.02621768, -0.02140872, ..., -0.03810406,
         -0.02376959,  0.01649384],
        [-0.01234972,  0.00150529,  0.01511236, ..., -0.02073779,
          0.0090019 ,  0.00086964],
        [ 0.00203347,  0.01825282,  0.03173146, ..., -0.01318896,
          0.01681314, -0.01373081]],

...,

[[ 0.00742659, -0.00477625, -0.02239722, ..., -0.01731521,
          0.01841538,  0.01798433],
        [ 0.01559777, -0.02657391,  0.00054658, ...,  0.00900136,
         -0.0157967 ,  0.02202146],
        [ 0.02034638, -0.02659588, -0.00532501, ...,  0.00019716,
         -0.03309927, -0.02475256],
        ...,
        [ 0.00637606, -0.02354819, -0.01059305, ..., -0.00469687,
         -0.02339374,  0.0128742 ],
        [ 0.00086939, -0.0116466 ,  0.01296971, ...,  0.02513519,
          0.02286913, -0.01149296],
        [ 0.00864135,  0.00573007, -0.00199565, ..., -0.00874565,
         -0.00141275,  0.00927982]],

[[-0.01335181, -0.00021832, -0.00915218, ..., -0.02494039,
         -0.013908  , -0.00494683],
        [ 0.00794976,  0.03978929, -0.00324192, ..., -0.00812777,
          0.02491831, -0.01019836],
        [ 0.0093718 , -0.01373968, -0.00493635, ...,  0.03200319,
          0.00222443, -0.0508735 ],
        ...,
        [-0.02443163,  0.01714601, -0.02545727, ...,  0.00177113,
         -0.02363153,  0.0238693 ],
        [-0.02297954,  0.04735063, -0.05361018, ...,  0.00994879,
         -0.04265187, -0.01899002],
        [-0.02156006,  0.00544838,  0.00973611, ..., -0.02133918,
         -0.00145689, -0.00232209]],

[[ 0.01742528, -0.00615192,  0.00217556, ..., -0.01009956,
          0.03087114, -0.02774436],
        [-0.02156098, -0.01062487,  0.00157245, ...,  0.00781925,
         -0.00496119, -0.0196279 ],
        [ 0.00954189,  0.0165382 ,  0.00890495, ..., -0.01508511,
         -0.00338188,  0.02362835],
        ...,
        [-0.00717626,  0.01471218,  0.00573334, ...,  0.00746967,
          0.00471901, -0.00187142],
        [-0.01044797,  0.03700609, -0.02869983, ..., -0.01165732,
         -0.00662046,  0.00494565],
        [-0.00374042, -0.00148811, -0.02519495, ...,  0.0100678 ,
         -0.00519629, -0.00252824]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.66908747, 0.46852782, 0.54715693, ..., 0.6616011 ,
         0.5656772 , 0.64159864],
        [0.48781356, 0.56635064, 0.42218527, ..., 0.58824074,
         0.5461354 , 0.5628913 ],
        [0.53050244, 0.7302693 , 0.53919846, ..., 0.5227843 ,
         0.74768186, 0.59701794],
        ...,
        [0.60210836, 0.51070195, 0.43162695, ..., 0.52467024,
         0.57141036, 0.54694223],
        [0.5599992 , 0.67426634, 0.61060834, ..., 0.58559686,
         0.5597936 , 0.65107906],
        [0.58436966, 0.56616336, 0.5600407 , ..., 0.58374053,
         0.51726466, 0.5558968 ]]], dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': array([0.35639364, 0.34132004, 0.3678308 , 0.29332113, 0.3548022 ,
       0.3196629 , 0.34824505, 0.28022683, 0.34007522, 0.35805207,
       0.3447933 , 0.36301646, 0.3236222 , 0.32413065, 0.33023334,
       0.28743845, 0.34377715, 0.3393906 , 0.33953124, 0.32523715,
       0.3169872 , 0.31388932, 0.38340393, 0.36926383, 0.34337118,
       0.31441355, 0.34962794, 0.3661602 , 0.31924438, 0.35972518,
       0.33061033, 0.29423946, 0.31563804, 0.34502977, 0.35413533,
       0.37948734, 0.36898082, 0.29570147, 0.37050843, 0.3258704 ,
       0.37426075, 0.3357056 , 0.27940238, 0.3129776 , 0.38641983,
       0.3623211 , 0.34729213, 0.34133965, 0.32058895, 0.34806046,
       0.36713976, 0.3717784 , 0.37632713, 0.3323835 , 0.38991728,
       0.3203513 , 0.35657376, 0.33545214, 0.35677052, 0.36801088,
       0.32906908, 0.2758519 , 0.33993274, 0.30850875, 0.3776776 ,
       0.32257536, 0.34567377, 0.33076546, 0.36784562, 0.3375391 ,
       0.3420226 , 0.33942178, 0.35138342, 0.33556384, 0.36306417,
       0.31934866, 0.37257168, 0.39450267, 0.32862386, 0.3617384 ],
      dtype=float32)
                },
                'post': {
                    'b': array([-0.00214796,  0.00832791,  0.01047556, ..., -0.00122844,
       -0.00554719, -0.00057771], dtype=float32),
                    'w': array([[[ 0.02869009,  0.00943764,  0.00531797, ..., -0.01740457,
          0.01775924,  0.00723355],
        [-0.00965818,  0.01898344,  0.00488442, ..., -0.00729101,
         -0.01743222,  0.00118863],
        [-0.01206351, -0.00496965,  0.00367228, ...,  0.0010526 ,
          0.00596434, -0.00036469],
        ...,
        [ 0.01421978,  0.01114153, -0.0083301 , ...,  0.00824197,
         -0.00754816, -0.01727102],
        [ 0.00025025,  0.00437635,  0.00784304, ...,  0.01374844,
         -0.0200068 , -0.01633633],
        [-0.00375637, -0.00395306, -0.0253886 , ..., -0.00194593,
          0.01819008,  0.00381707]],

[[ 0.01722552, -0.04244521, -0.00431258, ...,  0.00176786,
          0.00946331,  0.0056506 ],
        [-0.00853273, -0.01363218, -0.00275103, ..., -0.00359124,
          0.00393195, -0.01264656],
        [ 0.00877213, -0.00109135, -0.01062137, ..., -0.00146905,
          0.02529415,  0.0136782 ],
        ...,
        [-0.02666897, -0.02296833, -0.01044468, ..., -0.01648276,
          0.01371865, -0.01244286],
        [ 0.01340312, -0.00350007,  0.03224954, ..., -0.01274395,
         -0.00598174,  0.00228823],
        [ 0.01454473, -0.01207266,  0.00854119, ..., -0.00181971,
         -0.01390109,  0.01225779]],

[[ 0.00877117, -0.00512112,  0.01821755, ..., -0.02161116,
          0.00909464, -0.02308644],
        [ 0.0003034 , -0.00459854,  0.00713122, ..., -0.01406906,
          0.00403063,  0.00618289],
        [ 0.00501147,  0.00952181,  0.01877754, ...,  0.0096187 ,
         -0.01968547,  0.01148628],
        ...,
        [-0.01752177,  0.02695305,  0.00292111, ..., -0.01944762,
         -0.01285591,  0.02438922],
        [-0.00121914,  0.01070144, -0.0052263 , ..., -0.01241826,
          0.01963204, -0.00884132],
        [-0.02529257,  0.00458834, -0.01070961, ...,  0.01035225,
         -0.0124197 ,  0.00526103]],

...,

[[ 0.00479759, -0.01849564, -0.00189566, ...,  0.00635015,
          0.00209891,  0.02757171],
        [-0.00788427, -0.02478534,  0.00297381, ...,  0.01194904,
         -0.01297107, -0.00155885],
        [ 0.01407311,  0.01302377,  0.00726788, ...,  0.00704835,
          0.02416144, -0.00323654],
        ...,
        [-0.02289534,  0.01116681,  0.01138832, ..., -0.0049791 ,
          0.000293  ,  0.00615731],
        [ 0.01518761,  0.00336146,  0.00775993, ..., -0.03038795,
          0.01186396,  0.02178906],
        [ 0.00655701, -0.00427991, -0.03800484, ..., -0.01055264,
         -0.01793648, -0.00852037]],

[[ 0.01483402, -0.00082713, -0.02076811, ..., -0.02173715,
          0.00440963, -0.00200505],
        [ 0.01868831, -0.00620065, -0.00742687, ..., -0.00118868,
          0.00312127,  0.00671868],
        [ 0.02256813, -0.00368825,  0.02055129, ..., -0.00953372,
          0.00228053, -0.01352627],
        ...,
        [-0.02097259,  0.00859769, -0.01943693, ..., -0.00688402,
         -0.00889684, -0.02680231],
        [ 0.02332987, -0.00056472, -0.01278151, ..., -0.01858904,
         -0.0162829 , -0.00070958],
        [-0.004777  ,  0.00765541,  0.01331298, ...,  0.00266161,
         -0.0082578 , -0.00426244]],

[[ 0.00562998,  0.00938226,  0.00687099, ..., -0.00116021,
         -0.00542865,  0.00782787],
        [-0.00352626, -0.0139083 , -0.00634241, ..., -0.00363757,
          0.01029849, -0.00430407],
        [-0.00223516,  0.00650535, -0.02042501, ..., -0.0076793 ,
          0.00563121, -0.00039084],
        ...,
        [ 0.00772774, -0.00045712,  0.01417555, ...,  0.0134108 ,
         -0.01336487, -0.01822393],
        [ 0.00024862, -0.00368071, -0.00101053, ..., -0.0114745 ,
         -0.01521763,  0.02290301],
        [ 0.01499688, -0.02343859,  0.02381371, ...,  0.01503901,
          0.00494653,  0.01106588]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.47343343, 0.43476278, 0.46138683, ..., 0.44730166,
         0.43082294, 0.4473007 ],
        [0.46457586, 0.45872605, 0.45704225, ..., 0.451592  ,
         0.44423094, 0.43040162],
        [0.43918887, 0.44563764, 0.44464478, ..., 0.43176398,
         0.47099456, 0.45862064],
        ...,
        [0.43585154, 0.41244647, 0.4406566 , ..., 0.44029868,
         0.4403495 , 0.416385  ],
        [0.4489413 , 0.44047144, 0.45070192, ..., 0.4487874 ,
         0.46926212, 0.43413475],
        [0.38351107, 0.37874788, 0.43086883, ..., 0.43052238,
         0.3892617 , 0.4541286 ]]], dtype=float32)
                },
                'query': {
                    'b': array([[-4.92933625e-03, -3.36020924e-02, -1.64103545e-02, ...,
         2.68765818e-03, -1.06418999e-02,  1.27139585e-02],
       [-5.93224121e-03,  1.45464456e-02, -1.70037597e-02, ...,
        -3.68339213e-04, -1.01805395e-02,  5.22182696e-03],
       [ 5.34146652e-03, -3.20420414e-03, -1.42281828e-03, ...,
         1.20814871e-02, -7.36192043e-04, -1.14789819e-02],
       ...,
       [ 2.92606885e-03,  1.45223234e-02, -3.49404365e-02, ...,
        -2.47589555e-02, -2.33482718e-02,  1.60419587e-02],
       [ 3.34996567e-03,  6.13400573e-03, -7.76892819e-04, ...,
         7.41102593e-03,  1.08059179e-02,  4.80904151e-03],
       [-1.40242314e-03,  5.81266219e-03,  5.59961842e-03, ...,
         4.84774867e-03,  6.56471308e-03,  3.65962405e-05]], dtype=float32),
                    'w': array([[[ 0.00874558,  0.01253717, -0.0166835 , ..., -0.00588645,
         -0.01074735,  0.04966896],
        [ 0.0316819 ,  0.01123518, -0.04319673, ..., -0.06064183,
          0.00224262,  0.01926875],
        [ 0.00822638,  0.01188794, -0.01699981, ..., -0.03090369,
         -0.0030938 , -0.0177431 ],
        ...,
        [-0.02056288,  0.00035961, -0.02575049, ..., -0.06145574,
         -0.00127565,  0.01225445],
        [ 0.03344519, -0.01105584,  0.00055006, ..., -0.05779998,
         -0.01317679, -0.00418735],
        [ 0.04997033,  0.03178057,  0.00088188, ..., -0.02633874,
          0.01553694,  0.03623275]],

[[-0.01207857,  0.00423386,  0.06000669, ...,  0.03871098,
         -0.03341449, -0.01783747],
        [-0.0280232 , -0.00167071, -0.02085518, ..., -0.01455171,
          0.03510184, -0.00682487],
        [ 0.01640348,  0.03323187, -0.01176562, ..., -0.04087228,
          0.03157068,  0.01403674],
        ...,
        [-0.02498903,  0.00540706, -0.01717909, ...,  0.01587102,
         -0.00908483,  0.00381056],
        [ 0.00145994, -0.00954437, -0.00407664, ...,  0.00571248,
         -0.02605001,  0.01133845],
        [-0.01477077, -0.00498868,  0.00576666, ..., -0.00472735,
          0.00090954,  0.04035481]],

[[ 0.00064278,  0.00899595, -0.0002706 , ..., -0.02063922,
          0.02349149, -0.0031553 ],
        [-0.00797031,  0.0193837 , -0.02764669, ...,  0.01291416,
          0.00152847, -0.01024916],
        [ 0.01235074,  0.01203652, -0.00917403, ..., -0.02464635,
          0.01227925, -0.02705244],
        ...,
        [ 0.01193337, -0.0033837 , -0.02456444, ..., -0.00840133,
         -0.0208401 , -0.00999506],
        [ 0.00051914,  0.03385017,  0.00218612, ..., -0.00139479,
          0.0024665 ,  0.012002  ],
        [-0.00147951, -0.00100993,  0.03615292, ...,  0.0072919 ,
          0.01947811,  0.01589841]],

...,

[[ 0.01140319, -0.0004125 , -0.01638112, ..., -0.0497528 ,
          0.02180238, -0.0287857 ],
        [-0.00034311,  0.00881839, -0.0067967 , ...,  0.04089059,
         -0.0481857 , -0.00611395],
        [ 0.00279048,  0.0002274 , -0.02910166, ...,  0.02695685,
         -0.01528562,  0.01362267],
        ...,
        [ 0.01833747, -0.01239537, -0.01231164, ..., -0.0175549 ,
          0.02507422, -0.01443023],
        [ 0.03436172, -0.03231702,  0.01365453, ...,  0.01789769,
          0.03691792,  0.03619681],
        [ 0.01053255, -0.00864501, -0.00823828, ..., -0.00550314,
          0.01029504, -0.04074318]],

[[-0.01057536, -0.02479582, -0.01808418, ..., -0.0108863 ,
          0.03405898,  0.00468961],
        [ 0.00078452,  0.04046457, -0.00615619, ...,  0.03381054,
          0.01710905,  0.0083518 ],
        [ 0.01195209, -0.00779157,  0.04074396, ..., -0.0302161 ,
         -0.01856193, -0.04429938],
        ...,
        [ 0.0160401 , -0.0027195 , -0.00015057, ..., -0.03401104,
         -0.02881269, -0.01485565],
        [-0.00354678,  0.02478794,  0.0035091 , ..., -0.06007581,
         -0.03027204,  0.0517045 ],
        [-0.00797556, -0.02337614,  0.05513334, ..., -0.04051515,
         -0.04923127, -0.00632349]],

[[-0.03653891,  0.00268992,  0.00891523, ...,  0.02581696,
         -0.00660179,  0.02126133],
        [ 0.02626838,  0.01582966,  0.02101961, ...,  0.03602818,
         -0.02656418, -0.04796021],
        [ 0.00724903, -0.01323678,  0.01695194, ...,  0.03835869,
         -0.00470296,  0.03168933],
        ...,
        [ 0.00065492, -0.02142965,  0.00766696, ..., -0.01565139,
          0.00737353, -0.00363003],
        [-0.01310692, -0.01751375,  0.03384865, ..., -0.011557  ,
         -0.02706607, -0.0287535 ],
        [ 0.01559199,  0.0444673 , -0.03096942, ..., -0.00501271,
         -0.04978823, -0.03677486]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.7035102 , 0.8449393 , 0.85104895, ..., 0.7361709 ,
         0.62313074, 0.7997682 ],
        [0.8693186 , 0.94893444, 0.76424074, ..., 0.902621  ,
         0.9188678 , 0.845198  ],
        [0.77527416, 0.6781386 , 0.815862  , ..., 0.8827363 ,
         0.6725459 , 0.7756997 ],
        ...,
        [0.66895306, 0.82147855, 0.73996043, ..., 0.8679826 ,
         0.8052059 , 0.6401726 ],
        [0.7513392 , 0.7908496 , 0.91533977, ..., 0.7288982 ,
         0.8134705 , 0.8262175 ],
        [0.82732826, 0.7542405 , 0.82024944, ..., 0.84305584,
         0.84164965, 0.8915677 ]]], dtype=float32)
                },
                'value': {
                    'b': array([[-1.7844223e-04, -4.9078255e-03,  8.7939674e-04, ...,
        -1.0633421e-03,  4.2153242e-05,  8.6045424e-05],
       [ 3.3815745e-03, -3.3321648e-04,  2.6374005e-03, ...,
        -7.3969213e-04,  2.4472391e-03, -9.2108030e-04],
       [-1.4806272e-03, -7.8831380e-04, -3.1488092e-04, ...,
        -2.8324886e-03,  2.2519149e-03,  2.4730733e-03],
       ...,
       [ 1.5006368e-03,  1.1138931e-03,  1.2836054e-03, ...,
        -1.9483248e-03,  1.9670355e-03,  2.0739788e-03],
       [-1.1379643e-03,  3.1626232e-03, -2.6456336e-03, ...,
         6.7569410e-05, -6.4134382e-04,  7.1093894e-04],
       [ 2.6015060e-03,  2.1737472e-03, -6.5869543e-05, ...,
        -4.1303635e-03, -3.1118204e-03,  3.3902212e-03]], dtype=float32),
                    'w': array([[[-2.03157421e-02, -9.15903971e-03, -1.30444383e-02, ...,
          7.36718590e-04, -4.22519026e-03,  4.15073941e-03],
        [ 6.77502574e-03,  7.28660147e-04, -2.45866347e-02, ...,
         -9.14037600e-03,  8.25326890e-03, -8.17570277e-03],
        [-1.19243674e-02, -5.63951815e-03, -1.43306993e-03, ...,
          6.03869790e-03,  2.69319844e-02,  7.01575167e-03],
        ...,
        [-1.51856628e-03,  1.58385616e-02,  1.25889126e-02, ...,
         -4.13889857e-03, -2.59260560e-05,  9.66171175e-03],
        [-1.85218062e-02,  1.11629032e-02, -1.86361023e-03, ...,
          2.39219842e-03,  2.28052828e-02,  1.11672943e-02],
        [ 6.10933779e-03, -8.68937373e-03, -6.37223851e-03, ...,
          6.03761245e-03, -8.80723353e-03, -1.75700113e-02]],

[[-9.26308625e-04,  1.52963521e-02,  3.09755746e-03, ...,
          2.55698655e-02, -1.21019334e-02, -1.63840293e-03],
        [-5.89298457e-03,  9.04216524e-03,  1.73227061e-02, ...,
         -6.31362293e-03, -3.00470833e-02, -2.95150802e-02],
        [-4.51530237e-03,  1.12321908e-02, -1.89173445e-02, ...,
          1.14657788e-03, -7.98892323e-03,  9.58955847e-03],
        ...,
        [-3.07530677e-03,  2.33628340e-02, -2.92354659e-03, ...,
          2.11478975e-02,  3.35696409e-03,  2.00801194e-02],
        [-1.76166650e-02, -1.17907487e-02, -4.58410615e-03, ...,
         -1.04223378e-02,  1.35181984e-02,  6.31435961e-03],
        [-1.32341869e-02, -8.12042132e-03,  3.67030757e-03, ...,
          9.43860156e-04, -2.13353336e-02,  4.52915300e-03]],

[[-1.02022523e-02,  6.06267154e-03,  6.84145605e-03, ...,
          7.65452976e-04, -1.38395196e-02,  1.18651167e-02],
        [ 1.65014230e-02,  1.46501819e-02, -2.75398046e-02, ...,
          5.24902670e-03, -7.57085392e-03, -6.77489955e-03],
        [ 9.83063877e-03, -1.57569610e-02,  9.22523253e-03, ...,
          4.18421021e-03, -4.21982724e-03,  1.10386461e-02],
        ...,
        [ 9.82399361e-05, -2.66483589e-03,  9.17141233e-03, ...,
         -2.98011256e-03, -3.50138149e-03, -2.53317459e-03],
        [ 8.83841712e-04,  6.38461998e-03, -4.72358149e-03, ...,
          7.01155281e-03,  6.77048042e-03,  2.68369680e-03],
        [ 7.24163884e-03, -1.13477884e-02,  8.24804418e-03, ...,
         -2.56958161e-03, -9.22856666e-03, -1.54759493e-02]],

...,

[[ 1.24723446e-02,  1.11852689e-02,  4.79497202e-03, ...,
         -1.69358160e-02, -2.57418025e-02, -9.60162957e-04],
        [-1.10319089e-02,  1.30192386e-02, -1.51036689e-02, ...,
         -1.03058238e-02,  2.69386899e-02,  1.67487524e-02],
        [ 1.33864302e-02,  2.65204371e-03, -2.21515857e-02, ...,
         -3.23667075e-03,  5.85610094e-03, -1.92302130e-02],
        ...,
        [ 1.37856873e-02,  9.07670148e-03,  1.27143238e-03, ...,
         -3.36479163e-03,  1.49861304e-02, -2.32312689e-03],
        [-1.05997985e-02, -7.53253279e-03,  9.82390903e-03, ...,
          9.88049712e-03,  2.36325432e-03,  3.76692915e-04],
        [ 1.18984201e-03,  1.06759379e-02,  1.55002512e-02, ...,
          1.98383592e-02, -5.13773703e-04,  6.76507829e-03]],

[[-2.16461737e-02, -1.00604463e-02,  2.64870794e-03, ...,
         -7.83608668e-03,  6.58028221e-05, -7.79587729e-03],
        [ 5.30221500e-03,  4.31681005e-03, -9.90055688e-03, ...,
          1.83347869e-03, -6.94733951e-03,  1.15217557e-02],
        [ 2.02032160e-02, -8.53739399e-03,  7.68151483e-04, ...,
          7.91223545e-04,  5.60902990e-03,  1.94969773e-02],
        ...,
        [-2.92261411e-03, -1.95237983e-03, -1.49604967e-02, ...,
          7.39419693e-03,  3.65056284e-03, -8.30383599e-03],
        [ 4.48001130e-03,  4.04851278e-03, -5.52808284e-04, ...,
          3.37602347e-02,  3.28892912e-03, -1.76658146e-02],
        [-1.00997724e-02,  8.36819224e-03,  1.75992660e-02, ...,
          1.09374402e-02,  1.08999033e-02, -2.77742222e-02]],

[[-4.68146466e-02, -9.26453341e-03, -2.99443770e-02, ...,
          1.04604037e-02, -1.59611069e-02, -2.64821807e-03],
        [-1.03302562e-04, -6.50553359e-03, -4.10049642e-03, ...,
          2.07577683e-02,  1.37467496e-02, -7.90071487e-03],
        [ 2.25197431e-03, -1.07433237e-02,  5.16068377e-03, ...,
          2.34127864e-02, -1.93827450e-02,  3.20600392e-03],
        ...,
        [ 2.07280535e-02, -7.11975561e-04, -1.49935167e-02, ...,
          6.89100660e-03, -4.35881875e-03, -5.95880672e-03],
        [ 1.53170358e-02, -1.15952250e-02, -1.87348202e-02, ...,
         -2.00649239e-02,  1.02016167e-03, -6.05729036e-03],
        [-7.36127701e-03,  1.23309800e-02,  4.21048189e-03, ...,
         -2.26222374e-03,  1.46400416e-03, -1.72078945e-02]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.5044787 , 0.467246  , 0.511425  , ..., 0.47382325,
         0.44768572, 0.4746329 ],
        [0.4790215 , 0.4818172 , 0.4771302 , ..., 0.45695812,
         0.46395633, 0.4577731 ],
        [0.48133236, 0.503365  , 0.46920714, ..., 0.45855042,
         0.49380794, 0.49631616],
        ...,
        [0.4721078 , 0.47130093, 0.47319818, ..., 0.4826201 ,
         0.46746072, 0.45201886],
        [0.47881937, 0.49161127, 0.47914323, ..., 0.48661628,
         0.54340506, 0.4614618 ],
        [0.42754102, 0.4006873 , 0.46539077, ..., 0.4744432 ,
         0.42855835, 0.50393283]]], dtype=float32)
                }
            }
        },
        'x_layers_13': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': array([-0.0034478 , -0.0628284 , -0.00682131, ..., -0.01855724,
       -0.00795195, -0.01122809], dtype=float32)
                    },
                    'linear': {
                        'w': array([[-0.03151614, -0.0080314 , -0.04668951, ...,  0.01049254,
         0.03591558,  0.00079862],
       [-0.06258383, -0.04206429,  0.02148353, ...,  0.00677137,
         0.03879655, -0.02585962],
       [-0.03223332, -0.04780781, -0.02068976, ..., -0.01317758,
         0.01165592, -0.0696369 ],
       ...,
       [ 0.01789408,  0.01969726,  0.04556585, ...,  0.06506349,
         0.03052077,  0.06799204],
       [-0.01471116,  0.04009691,  0.03642584, ...,  0.02687883,
         0.00796544, -0.02481566],
       [ 0.04239961,  0.01176694,  0.04996835, ..., -0.05828528,
         0.02227721,  0.02466226]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.1060231, 2.4845655, 1.1867056, ..., 1.2604487, 1.1206441,
        1.2176853]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': array([ 1.7072462e-03, -8.4020954e-05,  3.6142219e-03, ...,
       -1.6132628e-03,  6.7724548e-05,  2.5193540e-03], dtype=float32)
                    },
                    'linear': {
                        'w': array([[ 0.01894921,  0.00237881,  0.02861445, ...,  0.05236435,
        -0.03568534, -0.02537734],
       [-0.01517698, -0.1817214 ,  0.00510799, ..., -0.1363022 ,
         0.07744772, -0.04587692],
       [-0.0719701 , -0.03862236,  0.00605929, ..., -0.0009384 ,
        -0.01894928,  0.0116022 ],
       ...,
       [ 0.02291214,  0.03107316,  0.0581912 , ..., -0.00307031,
        -0.01773866,  0.05663324],
       [-0.00443126, -0.03759074, -0.00107925, ...,  0.0357363 ,
        -0.0214347 , -0.01909814],
       [ 0.02096119,  0.00543365, -0.00405475, ..., -0.04826686,
        -0.03115325, -0.00325373]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.24842  , 1.311872 , 1.1905326, ..., 1.3474562, 1.2627182,
        1.5220968]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': array([-0.00180957,  0.01529409,  0.01958043, ..., -0.00829968,
       -0.02563367,  0.00275606], dtype=float32),
                    'scale': array([-0.0186807 ,  0.00872798,  0.02538491, ...,  0.0055809 ,
        0.02315959, -0.01200881], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': array([1.1680623, 1.157661 , 1.0796328, ..., 1.1467127, 1.1279683,
       1.1683893], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': array([[ 4.6610608e-04, -9.2302234e-04,  1.9256090e-04, ...,
        -1.8762015e-03,  3.1015230e-03,  7.7282086e-05],
       [ 4.2546907e-04,  1.2939374e-03,  4.0687714e-03, ...,
        -2.2190525e-03, -2.9033702e-03,  4.7080475e-04],
       [ 3.6429802e-03,  1.0424472e-03, -4.5762388e-03, ...,
         3.9652204e-03,  1.9283589e-03, -5.2442768e-04],
       ...,
       [-1.1196211e-03,  2.0621186e-03, -1.0337863e-03, ...,
         4.7227193e-04,  1.5723511e-03, -2.1845263e-03],
       [ 2.8016064e-03, -1.5755885e-03, -1.3314369e-03, ...,
         1.4067355e-03, -4.0574753e-04,  7.7799556e-04],
       [-3.2866246e-03,  4.4263457e-03, -3.2770564e-04, ...,
         1.1289970e-04, -3.6392491e-03,  6.3449412e-04]], dtype=float32),
                    'w': array([[[-1.08706683e-03,  4.60137837e-02,  8.90870579e-03, ...,
          1.76706687e-02,  9.42360982e-03,  2.26893439e-03],
        [-2.84814835e-02,  2.05505025e-02,  2.50078719e-02, ...,
         -8.25714320e-03, -1.54784461e-02,  1.57097764e-02],
        [ 2.61329673e-02, -1.43390916e-05, -1.57456137e-02, ...,
          5.81572438e-03,  9.63232853e-03,  3.29728536e-02],
        ...,
        [-2.58284789e-02, -6.61483314e-03,  4.17501107e-03, ...,
          1.56273600e-03, -1.34084104e-02, -2.00806791e-03],
        [ 2.15532375e-03,  5.60544012e-03, -1.21262446e-02, ...,
          1.41353793e-02,  1.00403710e-03, -2.26156227e-02],
        [ 6.50632137e-04,  3.77634016e-04,  9.30224918e-03, ...,
          2.15902999e-02, -2.99683325e-02,  4.09428496e-03]],

[[-5.27522899e-03,  2.87451968e-03,  1.06576411e-02, ...,
          1.07188504e-02, -2.51994492e-03, -3.28309438e-03],
        [-6.13255892e-03,  9.52015258e-03,  1.00164078e-02, ...,
         -9.81467217e-03, -1.30455401e-02, -4.91326318e-05],
        [ 1.51370522e-02, -7.62285432e-03,  2.52705123e-02, ...,
         -1.15954662e-02,  9.43033863e-03, -1.50437839e-02],
        ...,
        [-1.25185316e-02,  1.15327137e-02,  7.73496041e-03, ...,
         -6.85159070e-03, -5.90030104e-03,  1.32565210e-02],
        [-8.42716731e-03,  2.67493515e-03,  2.05571367e-03, ...,
          2.82730367e-02, -1.98101904e-02, -1.15765696e-02],
        [ 7.91658461e-03, -3.89121263e-03,  1.10226814e-02, ...,
          5.62335481e-04,  4.02511982e-03,  2.94603966e-03]],

[[ 8.03483650e-03, -2.33557522e-02, -1.38693722e-03, ...,
          1.83575545e-02,  1.91904884e-02, -1.53819667e-02],
        [ 1.23807024e-02, -1.67379230e-02, -1.76606104e-02, ...,
          7.21384911e-03,  3.21100503e-02,  8.89250357e-03],
        [-4.27075475e-03,  1.29289934e-02,  2.07111556e-02, ...,
         -3.78001877e-03,  5.15621714e-03, -1.84636228e-02],
        ...,
        [-7.28733605e-03,  1.22211445e-02,  1.15918927e-02, ...,
          2.58502141e-02,  9.47821978e-03,  1.60336252e-02],
        [-8.26664083e-03,  5.68870595e-03,  3.19347233e-02, ...,
         -4.69510956e-03,  1.38765387e-02, -3.25167179e-02],
        [ 3.06078475e-02, -9.79802944e-03,  3.17657627e-02, ...,
         -7.01859221e-03,  1.00923860e-02,  3.60700078e-02]],

...,

[[-8.43909627e-04,  3.61793186e-03,  1.28820781e-02, ...,
         -1.46371499e-02, -1.74430944e-02, -8.20774585e-03],
        [ 1.77437123e-02,  2.64214464e-02,  2.22612787e-02, ...,
         -9.30883456e-03, -2.24732817e-03, -1.52480770e-02],
        [-6.78731268e-03, -1.38723841e-02,  4.35494632e-02, ...,
         -2.12108754e-02,  2.64945608e-02, -2.25238465e-02],
        ...,
        [-9.24816355e-03,  1.97986085e-02, -6.95258100e-03, ...,
          2.68129129e-02,  1.15230707e-02,  1.48687549e-02],
        [ 9.45891999e-03, -3.69712216e-04, -7.51144998e-03, ...,
          7.82904401e-03,  4.60519828e-03, -1.37355654e-02],
        [-1.39810545e-02,  8.91153142e-03, -1.22041898e-02, ...,
         -1.65806841e-02, -2.67707207e-03,  5.80587471e-03]],

[[ 1.64198633e-02, -6.68989541e-03,  6.87183812e-03, ...,
          1.73081867e-02, -9.74146742e-03, -1.08897556e-02],
        [-2.38329973e-02,  2.73840385e-03, -2.69714650e-03, ...,
         -9.91132203e-03,  8.67062062e-03,  3.17646889e-03],
        [ 5.64266508e-03,  1.04839844e-03, -1.44311767e-02, ...,
          1.88627765e-02, -1.47463409e-02, -9.71260853e-03],
        ...,
        [ 1.56415366e-02,  7.77565656e-05,  6.42800005e-03, ...,
          1.60318508e-04,  1.10347513e-02,  1.69303343e-02],
        [-2.46376940e-03,  8.27714987e-03,  3.04045621e-03, ...,
         -7.15165306e-03, -2.62234313e-03, -6.02492550e-03],
        [ 2.16367505e-02,  1.38784613e-04,  4.50988635e-02, ...,
         -2.50860006e-02,  1.01543702e-02, -6.53972803e-03]],

[[ 7.57395895e-03, -8.69783759e-03, -6.51977910e-03, ...,
         -7.83248991e-03,  1.37400422e-02,  9.63847339e-03],
        [ 1.93790216e-02,  1.15005560e-02,  2.57335790e-03, ...,
         -1.17511407e-03, -1.32754035e-02,  2.84431316e-02],
        [ 8.90493859e-03, -6.35170203e-04,  1.79159716e-02, ...,
         -7.44165573e-03, -1.16720190e-02, -6.11115992e-03],
        ...,
        [-1.76805817e-02, -1.92710981e-02,  2.11877096e-02, ...,
         -1.47993183e-02, -2.90360246e-02, -2.44445037e-02],
        [ 2.60678609e-03,  1.54918600e-02,  7.54820416e-03, ...,
          1.60006043e-02, -3.96102592e-02, -1.15687342e-03],
        [ 1.01102330e-02, -1.23707214e-02, -7.94599764e-03, ...,
          4.61885557e-02, -6.15432067e-03,  2.03162176e-03]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.53228104, 0.5963369 , 0.53198034, ..., 0.5444237 ,
         0.57113814, 0.5069446 ],
        [0.74276155, 0.51038635, 0.47085488, ..., 0.52366704,
         0.45038044, 0.53787285],
        [0.50431734, 0.47188303, 0.54802424, ..., 0.51557297,
         0.5520455 , 0.67313933],
        ...,
        [0.58497614, 0.62775135, 0.4949539 , ..., 0.5082841 ,
         0.50935674, 0.5716479 ],
        [0.5130417 , 0.4756926 , 0.5876082 , ..., 0.60290116,
         0.6515314 , 0.52238375],
        [0.48854566, 0.48253092, 0.6112101 , ..., 0.5802609 ,
         0.54353136, 0.60679287]]], dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': array([0.3423228 , 0.37805688, 0.38570723, 0.3631645 , 0.3697752 ,
       0.38898537, 0.3462277 , 0.40135533, 0.32160336, 0.32755578,
       0.36695844, 0.38510692, 0.37241417, 0.36000612, 0.40425485,
       0.39355522, 0.37704387, 0.34560984, 0.39334768, 0.34970805,
       0.41583103, 0.32193702, 0.39897722, 0.38455772, 0.3881263 ,
       0.36545965, 0.30030486, 0.3719625 , 0.3724528 , 0.36381948,
       0.3207002 , 0.3917756 , 0.37410504, 0.35788256, 0.3723777 ,
       0.40420017, 0.30146837, 0.33636385, 0.37770233, 0.34213907,
       0.43725038, 0.35701525, 0.36562383, 0.32315183, 0.41536447,
       0.35835594, 0.34123057, 0.36961076, 0.32459491, 0.3978243 ,
       0.44356823, 0.41040212, 0.35807064, 0.37811923, 0.401245  ,
       0.38634533, 0.37138543, 0.33645675, 0.36295387, 0.35502782,
       0.36917937, 0.3897106 , 0.37134442, 0.39309275, 0.36695778,
       0.41020274, 0.44558018, 0.36960307, 0.29633558, 0.35949242,
       0.30078134, 0.37817764, 0.2984029 , 0.3855079 , 0.34376156,
       0.34066135, 0.34903902, 0.34249932, 0.35855636, 0.42332006],
      dtype=float32)
                },
                'post': {
                    'b': array([ 0.0003246 ,  0.00847099,  0.01464174, ..., -0.0033611 ,
       -0.00732129,  0.00176842], dtype=float32),
                    'w': array([[[-0.01333189, -0.00949195, -0.00053049, ...,  0.01240137,
         -0.00696176,  0.00108714],
        [-0.01398571, -0.00581668,  0.01127646, ...,  0.01062614,
          0.01003156, -0.0046383 ],
        [ 0.00261621, -0.00092907,  0.02294742, ..., -0.00945895,
          0.00738646,  0.01177557],
        ...,
        [ 0.03838847, -0.00842606, -0.00133155, ..., -0.00734077,
          0.01323657,  0.02828315],
        [-0.03245032,  0.00308438,  0.00897119, ...,  0.00759104,
         -0.00044007, -0.01147511],
        [ 0.00633265, -0.01913508,  0.00784844, ...,  0.00480823,
          0.00590358, -0.00663009]],

[[-0.00299581, -0.00517497, -0.00670307, ...,  0.00593408,
         -0.00332237,  0.0202051 ],
        [-0.01761871, -0.00733013, -0.00426498, ..., -0.00025184,
         -0.02124729,  0.02750478],
        [-0.01009283,  0.00274308,  0.00360174, ..., -0.00721292,
          0.02667607,  0.00796937],
        ...,
        [-0.01748743, -0.0108248 , -0.00127062, ...,  0.00585075,
         -0.02013801,  0.02031085],
        [ 0.00369795, -0.01433839,  0.01173536, ...,  0.00478352,
         -0.0245483 ,  0.02512564],
        [ 0.00472273,  0.00153474,  0.0078105 , ...,  0.0009682 ,
         -0.00131701,  0.02074756]],

[[ 0.01326769,  0.00603042, -0.01625889, ...,  0.02960653,
         -0.00335881,  0.00894076],
        [-0.01064683,  0.00013754, -0.00923376, ..., -0.02441404,
          0.00718767,  0.01418512],
        [-0.015638  ,  0.01246301,  0.00382231, ...,  0.01210467,
         -0.00757093,  0.02099155],
        ...,
        [-0.00591189,  0.00618605, -0.00617074, ...,  0.01182181,
         -0.02821254, -0.00071108],
        [-0.01192638,  0.01182433, -0.00947823, ...,  0.00591466,
          0.01700716,  0.02468597],
        [-0.01636377, -0.00094975,  0.00249019, ...,  0.0139281 ,
          0.02233758,  0.00633213]],

...,

[[-0.0170145 ,  0.01152131, -0.00434304, ..., -0.02416488,
          0.01287992,  0.00424458],
        [-0.01864726,  0.01006643,  0.01632176, ...,  0.00424803,
          0.01408952, -0.01048398],
        [-0.02251358, -0.01812711,  0.00131095, ...,  0.00199948,
          0.01198925,  0.00714228],
        ...,
        [-0.00919198, -0.00390246,  0.00771533, ..., -0.00388363,
          0.00095508, -0.02527265],
        [ 0.02789558,  0.02216544,  0.00098327, ...,  0.00256408,
         -0.01150388, -0.00541603],
        [-0.00530677, -0.01819404, -0.01334379, ..., -0.0059772 ,
         -0.00354978,  0.01075918]],

[[ 0.0182554 , -0.01828462,  0.00659558, ..., -0.00504959,
         -0.00578659, -0.03644215],
        [ 0.00366626, -0.01471603, -0.01019147, ..., -0.01839028,
          0.01397273,  0.02052104],
        [ 0.02750964,  0.02304667, -0.04257848, ..., -0.01364156,
          0.01307852,  0.00730396],
        ...,
        [ 0.01946117,  0.00414282, -0.00318116, ...,  0.00976689,
         -0.00175575, -0.01702852],
        [ 0.01807751, -0.0073047 ,  0.00811681, ..., -0.03190129,
          0.00495187,  0.0355152 ],
        [ 0.01347499, -0.02409538, -0.0129582 , ...,  0.00169767,
          0.00346595,  0.01452686]],

[[ 0.00438632,  0.00552457,  0.01929554, ...,  0.00988519,
         -0.00359951, -0.00395988],
        [-0.01838505, -0.01033139,  0.00202073, ..., -0.00086231,
          0.00257837, -0.00091347],
        [-0.01157259,  0.00156012,  0.01622672, ...,  0.00595554,
         -0.00686237,  0.00272262],
        ...,
        [-0.02153967,  0.01909429, -0.02143722, ..., -0.02285551,
         -0.00333509, -0.00466016],
        [-0.00993951,  0.01258533,  0.00091125, ...,  0.01065476,
          0.01349149,  0.01044108],
        [ 0.01078804,  0.02377654, -0.00526888, ..., -0.01481672,
          0.0023681 , -0.02840961]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.47289872, 0.42917073, 0.46712583, ..., 0.468564  ,
         0.46421725, 0.4703389 ],
        [0.45035943, 0.44936654, 0.46159878, ..., 0.45703077,
         0.4530067 , 0.43451935],
        [0.4628625 , 0.43478996, 0.45141703, ..., 0.46556103,
         0.45075977, 0.4478504 ],
        ...,
        [0.44069782, 0.44274935, 0.4716144 , ..., 0.43866012,
         0.45879117, 0.47706032],
        [0.39121628, 0.4814561 , 0.44583744, ..., 0.4753224 ,
         0.4541725 , 0.47319686],
        [0.40793547, 0.45419472, 0.4462662 , ..., 0.47833315,
         0.47363052, 0.43662354]]], dtype=float32)
                },
                'query': {
                    'b': array([[ 1.68616569e-03,  7.35044386e-03, -7.82678742e-03, ...,
         1.13712074e-02, -1.20169930e-02, -7.39986310e-03],
       [-4.70847776e-03,  6.84696948e-03,  1.45763187e-02, ...,
        -1.89887341e-02, -1.99760124e-02, -8.20217654e-03],
       [ 1.99295748e-02,  1.89537518e-02, -2.01193895e-02, ...,
         1.57649498e-02, -1.20130610e-02,  1.74873788e-02],
       ...,
       [-9.92237838e-05,  1.43216911e-03, -1.05841765e-02, ...,
         5.57100540e-03,  1.08484523e-02,  1.23333735e-02],
       [ 1.54685080e-02, -1.70757119e-02, -1.84352957e-02, ...,
         9.89105832e-03,  7.58174481e-03,  2.33456977e-02],
       [-8.85467511e-03,  6.02648500e-03, -4.42392053e-03, ...,
        -1.57583761e-03, -5.54478436e-04, -8.62513727e-04]], dtype=float32),
                    'w': array([[[ 0.03277196, -0.02859401, -0.00419887, ...,  0.03090124,
          0.05109725,  0.01152376],
        [ 0.00334124,  0.02527811,  0.02353178, ..., -0.00263442,
          0.00997576,  0.01351473],
        [-0.0116917 , -0.01465859, -0.00956394, ...,  0.01710192,
         -0.00084864,  0.01143387],
        ...,
        [-0.01677076, -0.00378386, -0.0149406 , ...,  0.00874252,
          0.02506459,  0.00360843],
        [ 0.01314708,  0.0301985 ,  0.00475084, ...,  0.02091082,
          0.03415568,  0.0144391 ],
        [-0.04189856,  0.02725412,  0.01354138, ...,  0.01726145,
         -0.06039976,  0.01708226]],

[[-0.00721604, -0.01122475,  0.00906932, ...,  0.01979994,
          0.01915505,  0.01679095],
        [-0.00823608, -0.02295914, -0.00024732, ...,  0.0291574 ,
         -0.0501032 , -0.00832714],
        [-0.00110509, -0.04844692, -0.00312842, ...,  0.00234616,
         -0.00022314,  0.00592112],
        ...,
        [-0.03135739,  0.0262335 , -0.00979377, ...,  0.01380539,
         -0.03992169,  0.03528836],
        [-0.02493829,  0.00080629, -0.01031507, ..., -0.05092132,
          0.02631439,  0.02859007],
        [-0.02136808, -0.01407127,  0.00108396, ..., -0.04483932,
          0.00690261, -0.00263384]],

[[ 0.02509242,  0.00068767, -0.02513626, ...,  0.00272727,
          0.00201789,  0.01799719],
        [-0.01684772,  0.00850673,  0.00068207, ...,  0.00247066,
         -0.01047844,  0.04504974],
        [-0.01106011,  0.00868868,  0.00304538, ...,  0.00737397,
         -0.0070529 ,  0.02826584],
        ...,
        [-0.00268174,  0.03360428,  0.0013599 , ...,  0.00681559,
          0.03629747,  0.00831298],
        [ 0.00621119, -0.05654305,  0.01545217, ..., -0.01338773,
         -0.04336489,  0.03081056],
        [-0.0013066 , -0.02572692,  0.00057133, ..., -0.00040557,
         -0.0038667 , -0.03006704]],

...,

[[ 0.00097242,  0.00575208, -0.00292164, ..., -0.04430099,
          0.00265532,  0.007615  ],
        [-0.00659848, -0.01168732,  0.01524789, ..., -0.02682199,
          0.01737099,  0.02003485],
        [ 0.03320982,  0.01940118,  0.03963336, ..., -0.01385362,
         -0.01110972,  0.00486455],
        ...,
        [-0.00413092,  0.01792374,  0.00578548, ..., -0.00270182,
          0.00759556, -0.06550304],
        [-0.01048632, -0.0144561 , -0.0269579 , ..., -0.01181703,
          0.00234207,  0.0063685 ],
        [ 0.03960787, -0.01332662, -0.01969515, ...,  0.07100058,
          0.02108816, -0.05444805]],

[[ 0.00724105, -0.01894746, -0.0087496 , ...,  0.00225464,
          0.03384326,  0.01377501],
        [-0.00447165,  0.02225273, -0.02554194, ..., -0.00094272,
         -0.02378647,  0.00841757],
        [ 0.0034877 , -0.01126554,  0.00230503, ..., -0.01673307,
         -0.00893878,  0.02590963],
        ...,
        [-0.02604976, -0.0117933 ,  0.01879387, ...,  0.00905253,
          0.0114352 , -0.0064004 ],
        [ 0.02724895,  0.00297151,  0.00447741, ...,  0.00148306,
         -0.00576708, -0.02083546],
        [ 0.01879717,  0.00488471,  0.01128259, ..., -0.02514385,
         -0.0317278 ,  0.00080828]],

[[-0.02100616, -0.01229226, -0.00664236, ..., -0.00397686,
         -0.01140649,  0.01656644],
        [ 0.02318282,  0.00089357,  0.00351443, ..., -0.04290025,
         -0.02527255,  0.02310306],
        [ 0.00123981,  0.00110993,  0.02510653, ..., -0.05794868,
          0.02559463,  0.03155119],
        ...,
        [ 0.0041406 ,  0.03817632,  0.0238045 , ..., -0.01445156,
         -0.02611304,  0.0221843 ],
        [-0.03239568, -0.03269725,  0.04132884, ...,  0.01554982,
          0.02150906, -0.0006056 ],
        [-0.00081417, -0.0145265 ,  0.0345278 , ...,  0.01734272,
         -0.02692604, -0.0183093 ]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.78208274, 0.8372392 , 0.7810194 , ..., 0.9301035 ,
         0.99357575, 0.8784337 ],
        [0.6553435 , 0.8105422 , 0.91441125, ..., 0.7932695 ,
         0.8616761 , 0.7169981 ],
        [0.88191336, 0.7470971 , 0.8440102 , ..., 0.724687  ,
         0.830646  , 0.7472641 ],
        ...,
        [0.7454711 , 0.78918797, 0.7672477 , ..., 0.78709835,
         0.74126357, 0.78932536],
        [0.85628814, 0.6301735 , 0.6638381 , ..., 0.7610328 ,
         0.7265357 , 0.69986457],
        [0.83489287, 0.7287642 , 0.80340904, ..., 0.8379677 ,
         0.8348412 , 0.64177424]]], dtype=float32)
                },
                'value': {
                    'b': array([[-6.0506293e-04,  5.6609279e-03,  1.6824390e-04, ...,
         3.6446922e-03, -4.2956732e-03,  2.7651975e-03],
       [-1.9702064e-03,  5.6249095e-04,  1.8836118e-04, ...,
        -1.1110047e-03, -7.1162550e-04, -1.0728554e-03],
       [ 1.7920202e-03, -7.7035325e-04, -1.0077038e-04, ...,
        -5.4486137e-04,  1.0511113e-03, -1.3385109e-03],
       ...,
       [ 1.4025952e-03, -5.3501499e-05,  3.7820532e-03, ...,
         7.0839765e-04,  1.5973275e-04, -7.4703916e-04],
       [ 2.3972094e-03,  1.8491159e-03, -2.3841648e-03, ...,
        -9.0237451e-04,  2.9343597e-03, -1.7331726e-03],
       [-2.2909330e-04,  3.7682767e-04,  6.2282931e-04, ...,
         7.3327468e-04, -1.5061294e-03,  6.9486239e-04]], dtype=float32),
                    'w': array([[[ 1.45941954e-02,  3.29494383e-03,  3.19986517e-04, ...,
         -9.69769619e-03,  1.06396377e-02, -1.90019403e-02],
        [ 1.43488497e-02, -9.36437212e-03,  4.04258724e-03, ...,
         -6.97107939e-03,  6.69743586e-03,  4.20573447e-03],
        [-2.41478905e-03,  1.75840594e-02, -1.30269798e-02, ...,
         -5.77327935e-03,  1.25292856e-02, -2.13064831e-02],
        ...,
        [-2.55727731e-02, -5.48362406e-03, -4.82275616e-03, ...,
         -1.88632980e-02, -7.25970929e-03,  1.63748935e-02],
        [-4.61501162e-03, -1.54769775e-02,  9.79034416e-03, ...,
         -3.15764919e-03,  1.46130370e-02,  1.14133917e-02],
        [ 2.69468836e-02, -1.04775866e-02, -8.37587751e-03, ...,
          1.32150073e-02, -1.61835197e-02,  1.91893261e-02]],

[[-1.31297195e-02,  2.23222487e-02,  2.69639827e-02, ...,
          5.25310542e-03,  2.34721089e-03, -1.49406036e-02],
        [-3.88548942e-03,  4.27055592e-03,  1.59564950e-02, ...,
         -7.51703093e-03,  1.59859378e-02,  2.42336513e-03],
        [-1.14898095e-02,  1.56635083e-02, -1.69220492e-02, ...,
         -9.85569973e-03,  6.86158333e-03, -1.49668017e-02],
        ...,
        [ 1.33758383e-02, -9.93928313e-03, -5.45043219e-03, ...,
         -1.61973387e-02,  3.01052793e-03,  2.30812356e-02],
        [-7.46069523e-03,  4.66854172e-03, -4.49841097e-03, ...,
          2.39429297e-04,  2.51386990e-03,  1.06357494e-02],
        [ 5.80206746e-03,  1.39213866e-02, -1.07834199e-02, ...,
         -1.87299885e-02,  2.92755867e-04, -1.61892660e-02]],

[[-1.06452722e-02, -4.89068404e-03,  5.59068797e-03, ...,
          1.16513604e-02, -7.74110248e-03, -1.11048929e-02],
        [ 1.75433215e-02,  6.57551782e-03, -1.79319456e-02, ...,
          9.54363868e-03, -6.53118361e-03, -1.21702105e-02],
        [-7.88253150e-04,  3.90434987e-03, -4.23693284e-03, ...,
          1.95031299e-03, -8.42137914e-03, -4.93645342e-03],
        ...,
        [-4.41000564e-03,  1.06764473e-02,  1.44665344e-02, ...,
          1.76251354e-03,  8.62312224e-03,  5.05607296e-03],
        [ 2.26121047e-03,  4.69154220e-05,  1.16300415e-02, ...,
          1.29878297e-02, -6.16351259e-04, -1.30897276e-02],
        [ 1.60715263e-03, -2.64528277e-03,  1.01121645e-02, ...,
          4.40053688e-03, -2.42634006e-02, -4.24439646e-03]],

...,

[[ 2.36383714e-02, -1.28607508e-02,  5.02150273e-03, ...,
          9.60131735e-03, -1.75693333e-02, -5.70473075e-03],
        [-1.39281142e-03, -1.27355773e-02, -2.32704580e-02, ...,
         -2.63107698e-02,  6.70625083e-03,  1.77655288e-03],
        [ 1.29360072e-02, -1.11631155e-02,  3.53246904e-03, ...,
         -1.91241887e-03, -7.07506435e-03,  2.61684903e-03],
        ...,
        [-2.25527585e-03, -8.81935563e-03, -1.65042058e-02, ...,
         -1.18197771e-02, -6.74883556e-03, -1.61913801e-02],
        [-2.31362991e-02, -6.17594738e-03,  2.26217695e-02, ...,
         -5.47542563e-03,  2.14086380e-02, -2.36117840e-02],
        [-2.95994617e-03,  7.01555470e-03,  1.08586475e-02, ...,
          1.19614033e-02,  1.28664449e-02,  1.82663556e-04]],

[[-4.22319630e-03,  8.78576841e-03, -2.30704751e-02, ...,
          1.67043786e-02,  5.37098246e-03,  6.56566303e-03],
        [ 1.93355652e-03, -5.65552199e-03, -3.48797324e-03, ...,
         -1.25056682e-02, -3.35406065e-02, -5.37997950e-03],
        [ 3.78591521e-03, -6.83129998e-03, -5.24395704e-03, ...,
          7.23225018e-03,  1.20409788e-03, -4.49350773e-04],
        ...,
        [ 5.45438845e-03, -5.76021895e-03, -1.00290552e-02, ...,
         -1.59054101e-02,  2.19838899e-02, -5.50129870e-03],
        [-2.69295793e-04,  2.72868876e-03,  6.65305415e-03, ...,
          1.01495106e-02, -5.70588093e-03, -5.31926937e-03],
        [-3.50542292e-02,  1.17312344e-02,  1.28120771e-02, ...,
          2.15886720e-02,  1.51882283e-02,  7.71389715e-03]],

[[ 1.83889840e-03, -9.06582084e-03,  8.98056477e-03, ...,
         -3.42143402e-02,  1.34969996e-02, -2.68482720e-03],
        [ 1.95417255e-02,  8.69073067e-03, -1.31595889e-02, ...,
         -3.40937357e-03,  1.11017330e-03, -2.54486352e-02],
        [ 4.01821807e-02, -1.65785868e-02, -1.30676490e-03, ...,
          9.75302607e-03,  8.82033538e-03, -1.49140591e-02],
        ...,
        [-5.97554166e-03,  2.31686104e-02,  1.02580320e-02, ...,
          6.09135488e-03,  1.25123495e-02, -2.34593637e-02],
        [ 1.90906879e-02, -3.57933566e-02, -4.43168171e-02, ...,
         -2.67305952e-02,  5.62132103e-03, -9.73594468e-03],
        [-5.58904791e-03,  1.43911988e-02, -5.09532820e-03, ...,
          2.70975586e-02,  2.44144676e-03,  9.71086882e-03]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.51124746, 0.44954726, 0.46779552, ..., 0.48364   ,
         0.47332934, 0.49376032],
        [0.46772793, 0.46325716, 0.46745098, ..., 0.46411797,
         0.47964695, 0.46030474],
        [0.4971136 , 0.4509621 , 0.48940882, ..., 0.47860518,
         0.48367563, 0.4734954 ],
        ...,
        [0.47490525, 0.457185  , 0.49063876, ..., 0.47890535,
         0.49498227, 0.47053412],
        [0.3958494 , 0.5141156 , 0.46219933, ..., 0.46747127,
         0.5064879 , 0.5051702 ],
        [0.433909  , 0.47445717, 0.47724828, ..., 0.5117322 ,
         0.48292512, 0.45668086]]], dtype=float32)
                }
            }
        },
        'x_layers_14': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': array([ 0.00029601, -0.01084311, -0.00675669, ..., -0.01585302,
       -0.01105882, -0.00785637], dtype=float32)
                    },
                    'linear': {
                        'w': array([[ 0.00582105,  0.01318311,  0.03708626, ..., -0.01971904,
        -0.01395077,  0.02952531],
       [ 0.00619035, -0.03169769,  0.01273892, ...,  0.01644244,
         0.01044564, -0.03745968],
       [-0.03273934,  0.0426847 ,  0.01360384, ..., -0.03426943,
        -0.03494504, -0.04589375],
       ...,
       [-0.02636916,  0.06461645,  0.00987081, ...,  0.00360583,
         0.01588036, -0.01292699],
       [ 0.02911054,  0.08070075,  0.03281379, ...,  0.02382054,
        -0.00493196,  0.01056881],
       [ 0.03009207, -0.04479855,  0.00666338, ...,  0.04239031,
         0.02533692,  0.023514  ]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[0.95773244, 1.186938  , 1.1175877 , ..., 1.1598177 , 1.0978218 ,
        1.0929062 ]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': array([ 0.00158921,  0.0018311 ,  0.00338582, ..., -0.00117   ,
       -0.00201284, -0.00039685], dtype=float32)
                    },
                    'linear': {
                        'w': array([[ 0.01624611,  0.0337978 ,  0.02114537, ..., -0.00867737,
        -0.02622462, -0.00260242],
       [ 0.01332312,  0.11018921, -0.04020173, ...,  0.00130645,
         0.04434073,  0.00471618],
       [ 0.03531237,  0.02962698, -0.02800041, ...,  0.00812992,
         0.05103358,  0.02820559],
       ...,
       [-0.05357926, -0.04292846,  0.01705871, ...,  0.02818025,
        -0.0585423 , -0.01409262],
       [ 0.02043611, -0.06592052, -0.02197525, ...,  0.01451133,
        -0.03996396, -0.02293865],
       [-0.02165488,  0.05503482,  0.06325393, ..., -0.02189652,
         0.01855572,  0.01316425]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.2462063, 1.3154709, 1.1777911, ..., 1.2787604, 1.2724371,
        1.3852394]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': array([ 0.00385938,  0.01357866,  0.02887791, ..., -0.00625249,
       -0.02258822,  0.01229282], dtype=float32),
                    'scale': array([-0.01239096,  0.0149999 ,  0.01164342, ...,  0.01213837,
        0.03059725,  0.00806229], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': array([1.0972409, 1.1051496, 1.1049464, ..., 1.121043 , 1.1114082,
       1.118717 ], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': array([[ 1.25841633e-03,  1.55034452e-03,  1.08872564e-03, ...,
        -9.54594580e-04, -1.65263680e-03, -4.30746097e-03],
       [-1.98853505e-03,  6.85305131e-05, -1.63052906e-03, ...,
         4.81848343e-04,  2.22792180e-04, -1.81035930e-03],
       [ 5.71344688e-04,  1.74748956e-03,  1.30943884e-03, ...,
        -4.07261134e-04, -1.63442385e-03,  8.52171797e-04],
       ...,
       [ 1.20874020e-05,  1.84527622e-03,  8.76582169e-04, ...,
        -6.50884467e-04,  2.91718403e-03, -2.68512813e-04],
       [ 1.46702142e-03,  1.36173528e-03, -1.60505017e-03, ...,
         1.72068970e-03, -3.37592070e-03,  1.48777233e-03],
       [-2.02768855e-03,  1.06071595e-04,  9.18209844e-04, ...,
        -1.73310947e-03, -8.43419926e-04, -2.83935195e-04]], dtype=float32),
                    'w': array([[[ 0.01589175,  0.03752959,  0.00500297, ..., -0.02840356,
         -0.01997883, -0.02230303],
        [ 0.02896536, -0.00461972, -0.00667112, ..., -0.02033726,
          0.01411753,  0.01882261],
        [-0.00266081,  0.00936987,  0.0201344 , ...,  0.00801549,
          0.00261194,  0.01681161],
        ...,
        [ 0.00451914, -0.0110497 ,  0.00833845, ...,  0.00385557,
         -0.00774913,  0.02446727],
        [ 0.03294024,  0.01529131, -0.02611599, ...,  0.01307458,
         -0.01864656,  0.00500216],
        [ 0.00635432,  0.03710559,  0.0123241 , ...,  0.01083144,
          0.0166588 , -0.00347141]],

[[-0.03525248, -0.02983907,  0.02150618, ..., -0.00144732,
          0.00498221,  0.00543415],
        [ 0.01448555, -0.00555849,  0.00365228, ...,  0.00392647,
         -0.01279497,  0.00785459],
        [ 0.0086551 , -0.00420757,  0.00674637, ...,  0.02207861,
         -0.02594632,  0.01674281],
        ...,
        [-0.03854414,  0.00527545,  0.03507176, ...,  0.0268054 ,
          0.03300948, -0.03546853],
        [-0.00323089,  0.01665277, -0.01082033, ...,  0.01212806,
         -0.02213307,  0.00897749],
        [ 0.00432751,  0.07296949, -0.00479869, ...,  0.02670059,
         -0.00987142, -0.003652  ]],

[[ 0.00522438, -0.01008221, -0.00522995, ..., -0.02711505,
         -0.00412196, -0.01753574],
        [ 0.02814708,  0.000779  , -0.01128127, ...,  0.01203646,
          0.00097294,  0.03949527],
        [ 0.00624943, -0.01234711, -0.00051581, ..., -0.02486076,
         -0.02476037, -0.02539629],
        ...,
        [-0.0083427 ,  0.00457395,  0.01165879, ...,  0.04828711,
         -0.03822126, -0.01611185],
        [ 0.01797941,  0.02932937,  0.01377241, ..., -0.00502206,
         -0.01840937,  0.01820203],
        [ 0.00458969, -0.03552117,  0.00842323, ..., -0.01047418,
          0.01426457, -0.01079076]],

...,

[[ 0.00337446, -0.00494773,  0.00232644, ..., -0.03967002,
          0.02315594, -0.01058441],
        [-0.00374536, -0.02243059,  0.0129877 , ..., -0.00215712,
         -0.0193033 , -0.00105581],
        [-0.01417497,  0.02113594,  0.01165939, ...,  0.00882702,
         -0.02012151,  0.01220951],
        ...,
        [-0.01551167,  0.0045996 ,  0.02410888, ..., -0.01052137,
          0.00344525,  0.00870658],
        [ 0.00020753,  0.05579822,  0.02898848, ..., -0.00173013,
          0.00848756,  0.01064574],
        [-0.0042817 ,  0.00619935,  0.00165318, ..., -0.02513871,
          0.01222925, -0.00750665]],

[[-0.01738883, -0.01238909, -0.01269979, ...,  0.00611273,
         -0.0031046 , -0.00508836],
        [ 0.00435876,  0.00716628,  0.00441743, ..., -0.02635167,
          0.00134546, -0.01466905],
        [ 0.03483389, -0.00310064,  0.04916606, ...,  0.03621708,
         -0.01160908, -0.05037719],
        ...,
        [ 0.02773033,  0.03806577,  0.00016818, ..., -0.01273797,
         -0.01533793,  0.0106364 ],
        [ 0.0135034 ,  0.0124661 , -0.00906702, ..., -0.00137257,
          0.00951501,  0.01415114],
        [-0.00523764,  0.00588071,  0.00563346, ..., -0.03795779,
         -0.02123829,  0.00739249]],

[[ 0.00333968,  0.02958334, -0.00317894, ...,  0.00457225,
         -0.0026831 , -0.00992109],
        [-0.02278157,  0.0058978 ,  0.02510419, ..., -0.01814865,
          0.00928549,  0.02088999],
        [-0.01182269, -0.00030995, -0.0023372 , ...,  0.00272881,
         -0.00585036,  0.02321105],
        ...,
        [ 0.00483019,  0.02018814,  0.02192444, ..., -0.02810882,
         -0.00201883, -0.00477643],
        [-0.0016086 ,  0.00650351, -0.02945073, ...,  0.01685349,
         -0.01316662,  0.00660059],
        [ 0.03185076, -0.02384418,  0.00400093, ...,  0.00800042,
          0.00193645, -0.01411922]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.64242554, 0.5794416 , 0.63296   , ..., 0.56769866,
         0.5795986 , 0.49113575],
        [0.5286971 , 0.5755267 , 0.4725382 , ..., 0.616967  ,
         0.5901392 , 0.61268353],
        [0.59386927, 0.5491543 , 0.6326702 , ..., 0.59499615,
         0.53190804, 0.6551641 ],
        ...,
        [0.5707093 , 0.7721315 , 0.62505585, ..., 0.55419594,
         0.5576113 , 0.5603438 ],
        [0.5810785 , 0.5320971 , 0.48676312, ..., 0.4718737 ,
         0.4609685 , 0.49673036],
        [0.54924357, 0.72018737, 0.5543811 , ..., 0.5999892 ,
         0.6006933 , 0.59968686]]], dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': array([0.37061766, 0.38822898, 0.35322222, 0.34075966, 0.2893336 ,
       0.33207327, 0.36952117, 0.31159094, 0.40695235, 0.3650621 ,
       0.30505344, 0.32530743, 0.38286367, 0.30215907, 0.37921858,
       0.31768548, 0.35374323, 0.27960575, 0.33220676, 0.27091444,
       0.33243063, 0.35063368, 0.33900893, 0.3868279 , 0.35574022,
       0.3405869 , 0.3743824 , 0.30515498, 0.38332865, 0.3609494 ,
       0.3061473 , 0.33329   , 0.3651566 , 0.3100548 , 0.3234738 ,
       0.3393868 , 0.37142864, 0.35722014, 0.30790803, 0.3584052 ,
       0.3168645 , 0.30959433, 0.25981882, 0.31101865, 0.32634166,
       0.38462982, 0.31919768, 0.29806888, 0.35299852, 0.37269595,
       0.35165468, 0.41325662, 0.3530622 , 0.36405906, 0.31029075,
       0.38071197, 0.38286918, 0.33056083, 0.35136825, 0.26401302,
       0.36774495, 0.32272413, 0.3591407 , 0.26142058, 0.26791415,
       0.3675712 , 0.39433083, 0.4035874 , 0.38473964, 0.37995642,
       0.31737867, 0.4083564 , 0.37195632, 0.36349788, 0.34845665,
       0.32651016, 0.36170384, 0.29348466, 0.35695028, 0.35530826],
      dtype=float32)
                },
                'post': {
                    'b': array([ 0.00137819,  0.00906399,  0.01699077, ..., -0.00279563,
       -0.00908101,  0.00332175], dtype=float32),
                    'w': array([[[ 0.0077813 , -0.0187567 ,  0.00135137, ..., -0.0074239 ,
         -0.00509173,  0.0028086 ],
        [-0.01502111,  0.01735665, -0.01783477, ..., -0.01821919,
          0.02553056, -0.0240744 ],
        [ 0.02150158, -0.00834437, -0.01518894, ...,  0.00089701,
         -0.00358897, -0.01120061],
        ...,
        [-0.01904649,  0.01619412, -0.03076166, ...,  0.00734015,
         -0.02245972, -0.00764283],
        [-0.00526892,  0.00547887, -0.0033242 , ...,  0.00683106,
          0.00080588,  0.01187587],
        [-0.00852314, -0.01253605, -0.01547347, ..., -0.004784  ,
         -0.01609213, -0.00491303]],

[[ 0.02101544,  0.01730167,  0.02191395, ...,  0.00150438,
          0.00869477, -0.01959719],
        [ 0.0053109 , -0.00984174,  0.01618154, ...,  0.01014376,
         -0.00402837, -0.00496181],
        [-0.00041001,  0.01650357, -0.00918049, ..., -0.01724378,
          0.01453209, -0.00450411],
        ...,
        [ 0.01052063,  0.01134723,  0.00399059, ..., -0.03812648,
         -0.02517106, -0.00583351],
        [ 0.00329291, -0.02358575,  0.00496385, ..., -0.01112499,
         -0.00742153, -0.01080149],
        [ 0.00766503,  0.0225117 ,  0.00251843, ...,  0.01661835,
          0.0248128 ,  0.00402973]],

[[-0.00103187, -0.02271016, -0.00680307, ..., -0.00873298,
          0.01281762,  0.01625166],
        [-0.00438799, -0.00785813, -0.01109597, ..., -0.02488057,
         -0.00409368, -0.00934564],
        [ 0.024538  , -0.00357003,  0.00078015, ...,  0.0091102 ,
         -0.00153082, -0.01972952],
        ...,
        [-0.01192023,  0.02724106, -0.00842953, ..., -0.01404229,
         -0.00200893, -0.00752169],
        [ 0.01564243,  0.02643112,  0.01525206, ..., -0.01202593,
         -0.01305926,  0.0010337 ],
        [ 0.0199012 , -0.00188075, -0.00743003, ...,  0.02620444,
          0.02550115, -0.01756352]],

...,

[[-0.01199227,  0.00871454,  0.011129  , ...,  0.00402309,
          0.00570987, -0.00687773],
        [ 0.02190694, -0.00711192, -0.00193789, ...,  0.00110549,
         -0.00788214,  0.0013736 ],
        [ 0.01646552,  0.00144623,  0.00198235, ..., -0.01052912,
          0.0188547 ,  0.00391463],
        ...,
        [ 0.00945043,  0.02376678,  0.00382838, ..., -0.00935743,
          0.00274879,  0.00801133],
        [ 0.01662589, -0.01179716,  0.00211741, ..., -0.00724348,
         -0.01273815,  0.00425153],
        [-0.00203212,  0.01654251,  0.01085144, ...,  0.0038071 ,
         -0.01050337, -0.0097528 ]],

[[-0.01421311,  0.02259041, -0.00900803, ...,  0.00902514,
          0.00418852, -0.02142691],
        [ 0.01666445,  0.00121269, -0.00789524, ..., -0.01556215,
          0.00236122,  0.02350651],
        [ 0.00196271, -0.00189148, -0.00099977, ...,  0.01035162,
         -0.00269132, -0.0094261 ],
        ...,
        [ 0.01478543,  0.00924319, -0.01727964, ..., -0.01104998,
          0.00154679, -0.02577997],
        [-0.00376452, -0.00292013, -0.02046316, ..., -0.01985718,
          0.01155387,  0.00559299],
        [ 0.0107471 ,  0.01140142, -0.00164137, ..., -0.01726211,
         -0.00668458, -0.00601897]],

[[ 0.01126035,  0.01273246, -0.01011568, ..., -0.01767948,
         -0.00159439, -0.0073643 ],
        [ 0.0052752 , -0.00572318, -0.01242626, ...,  0.00613892,
         -0.03157397,  0.00018649],
        [-0.00516451, -0.00544372,  0.00276264, ...,  0.00999606,
         -0.01028886, -0.00110074],
        ...,
        [ 0.01464988, -0.00893058, -0.00986978, ..., -0.00508787,
         -0.02117784, -0.0030193 ],
        [-0.00746697, -0.01532643, -0.00941347, ..., -0.00074038,
         -0.00212986,  0.02092797],
        [-0.01018735,  0.01143691,  0.01311445, ..., -0.01059256,
         -0.01391992,  0.00311824]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.45041078, 0.46959144, 0.44235015, ..., 0.44134474,
         0.47642964, 0.48616824],
        [0.47195742, 0.48970535, 0.49018347, ..., 0.48477077,
         0.5044727 , 0.47668937],
        [0.43908077, 0.46791545, 0.387303  , ..., 0.48213562,
         0.46331868, 0.47594944],
        ...,
        [0.45131445, 0.46695685, 0.41730383, ..., 0.46460035,
         0.46692842, 0.47296554],
        [0.43628243, 0.4408122 , 0.4292478 , ..., 0.44581342,
         0.4403963 , 0.4417896 ],
        [0.4483517 , 0.4609275 , 0.47700804, ..., 0.43639326,
         0.41476956, 0.4389742 ]]], dtype=float32)
                },
                'query': {
                    'b': array([[ 0.005839  ,  0.00983257,  0.0143399 , ..., -0.01401017,
        -0.02116291, -0.02012843],
       [ 0.01007332,  0.00880442,  0.01877066, ...,  0.00415033,
        -0.01021724,  0.00293849],
       [-0.00301371, -0.00309211, -0.00566929, ...,  0.00209435,
        -0.00242971,  0.00093665],
       ...,
       [ 0.00573737,  0.00622756,  0.00125796, ..., -0.01130205,
         0.01154514,  0.00515189],
       [ 0.01658523,  0.00806783, -0.01109188, ...,  0.01466294,
        -0.0154193 ,  0.01680345],
       [ 0.00661552, -0.00166651, -0.00556861, ...,  0.00759342,
         0.00582669, -0.00205902]], dtype=float32),
                    'w': array([[[ 4.31832578e-03,  4.43392396e-02, -1.94164272e-02, ...,
         -1.54080531e-02, -2.04299670e-02, -5.17385080e-03],
        [ 1.04202796e-02, -3.61938798e-03,  2.09692549e-02, ...,
          9.26325377e-03, -1.67021919e-02,  1.68145522e-02],
        [ 1.27045680e-02,  2.11668964e-02, -1.38618518e-02, ...,
         -2.85082720e-02,  4.86320928e-02, -2.15413631e-03],
        ...,
        [-9.61130206e-03, -1.08617987e-03,  4.71109618e-03, ...,
          1.04664522e-03,  6.03351854e-02, -5.40138781e-03],
        [ 2.92769670e-02,  3.52728590e-02, -3.96639816e-02, ...,
         -1.35405455e-02, -8.60308576e-03,  2.50309445e-02],
        [-2.57780645e-02, -2.47719996e-02,  7.15856850e-02, ...,
         -1.76313594e-02, -3.09293456e-02,  1.78850945e-02]],

[[ 2.47390997e-02,  5.65293105e-03, -1.60987992e-02, ...,
         -5.66915534e-02, -1.51922842e-02, -5.83268777e-02],
        [-2.64417636e-03,  3.17426473e-02,  2.08462179e-02, ...,
          1.57627128e-02, -3.67227606e-02, -4.39150035e-02],
        [ 3.33051500e-03,  2.79763155e-02,  6.52864203e-03, ...,
         -4.60573239e-03,  6.15050131e-03,  2.81052701e-02],
        ...,
        [ 1.49639342e-02,  3.65525261e-02,  2.86534801e-02, ...,
          3.85402814e-02,  3.83723713e-02, -2.05098502e-02],
        [ 1.23497946e-02, -3.31008136e-02, -1.64650120e-02, ...,
          4.42989841e-02,  1.43406810e-02,  4.60568778e-02],
        [ 3.00253499e-02, -3.20079271e-03, -8.82569142e-03, ...,
          1.50561268e-02, -8.84096418e-03, -2.02828310e-02]],

[[ 2.24574748e-02,  1.96464919e-02, -3.42796813e-03, ...,
         -6.79734051e-02, -7.87650794e-03, -1.81626864e-02],
        [ 4.98392694e-02, -2.63143070e-02, -1.64950043e-02, ...,
         -3.51649597e-02,  5.59684541e-03, -6.00041931e-05],
        [-2.81742588e-03, -1.43649662e-03, -1.26607371e-02, ...,
         -2.38804072e-02,  4.42015417e-02,  3.07527427e-02],
        ...,
        [ 3.41069861e-03,  1.13542331e-02, -1.47044277e-02, ...,
         -3.65831852e-02, -1.85070764e-02,  3.16809304e-03],
        [ 1.75473504e-02, -1.16419485e-02,  4.07463079e-03, ...,
          5.03172390e-02, -1.69017876e-03,  1.87073220e-02],
        [-1.42812217e-02,  7.68110994e-03, -2.11924575e-02, ...,
          3.04143801e-02, -3.18563096e-02, -4.87193745e-03]],

...,

[[-6.21698201e-02,  3.00060678e-02,  3.99990454e-02, ...,
          1.27259661e-02, -2.16455460e-02,  6.22699037e-03],
        [ 1.26074348e-03, -1.16656180e-02, -1.91802718e-02, ...,
          1.17440037e-02,  9.53414571e-03,  3.72846276e-02],
        [ 1.38570629e-02, -6.38084370e-04,  5.33942729e-02, ...,
         -1.80969760e-02,  1.54469796e-02,  2.65894327e-02],
        ...,
        [ 6.42145351e-02, -1.87670663e-02,  1.26777189e-02, ...,
          2.50951946e-02, -2.64055543e-02,  9.42695234e-03],
        [-3.45227146e-03, -2.31616553e-02,  1.79782826e-02, ...,
          1.82804018e-02,  2.41125971e-02,  7.53489975e-03],
        [-1.01818470e-03, -1.71474386e-02, -5.46405371e-03, ...,
          7.72300689e-03, -5.96350357e-02, -1.02749895e-02]],

[[-1.56565215e-02,  2.01507881e-02,  2.89510633e-03, ...,
         -2.32250597e-02, -9.01773665e-03, -2.06077006e-02],
        [-2.88240854e-02, -5.79046225e-03,  1.78494472e-02, ...,
         -3.92777007e-03, -7.22291041e-03,  1.27372183e-02],
        [-1.69353969e-02, -1.56654753e-02, -8.97205842e-04, ...,
         -4.45940904e-02, -2.08338667e-02, -3.71313244e-02],
        ...,
        [ 2.24471046e-03, -8.75685550e-03, -6.72053255e-04, ...,
         -1.80401020e-02, -3.54620330e-02, -1.71075854e-02],
        [-2.94765737e-02, -3.34563255e-02, -1.41663915e-02, ...,
          3.73317413e-02,  6.57628570e-03,  1.13215568e-02],
        [-1.43873170e-02, -3.71560990e-03,  8.71734135e-03, ...,
         -2.53836811e-03, -1.51491184e-02, -2.71045193e-02]],

[[ 1.74412504e-02,  2.77380887e-02,  2.66704592e-03, ...,
          1.13621624e-02,  3.15764658e-02,  1.05727613e-02],
        [-2.83396821e-02, -2.88270274e-03,  2.99232360e-02, ...,
          6.64325291e-03,  2.54750084e-02,  1.79602876e-02],
        [-4.25079698e-03,  1.65656265e-02, -5.48844039e-02, ...,
          4.05983292e-02,  2.32505482e-02,  2.22766977e-02],
        ...,
        [-3.46966344e-03, -4.04150458e-03, -5.48522398e-02, ...,
         -1.38870236e-02,  2.99531091e-02,  5.51477168e-03],
        [-2.55604777e-02,  1.75041482e-02,  2.97506843e-02, ...,
         -2.03855336e-02, -9.80742552e-05,  3.82814668e-02],
        [ 5.42269973e-03, -8.90803058e-03, -1.50118424e-02, ...,
         -2.57738512e-02, -2.58772653e-02,  8.77469871e-03]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.89834404, 0.96501297, 0.89159894, ..., 0.9307708 ,
         0.9834283 , 0.8508191 ],
        [0.93372285, 0.7044562 , 0.81689465, ..., 0.78034514,
         0.81813973, 0.8078872 ],
        [0.9834947 , 1.0560204 , 0.9396346 , ..., 0.92710227,
         1.1537861 , 0.8445274 ],
        ...,
        [0.7043108 , 0.6004386 , 0.8232796 , ..., 0.9337106 ,
         0.83005977, 0.7589937 ],
        [0.74351394, 0.7008312 , 0.7255922 , ..., 0.77499825,
         0.6073632 , 0.90837294],
        [0.8132381 , 0.6770858 , 0.81394166, ..., 0.7610588 ,
         0.84081435, 0.82759464]]], dtype=float32)
                },
                'value': {
                    'b': array([[-3.8052234e-03,  3.6348451e-03, -2.1750149e-03, ...,
         3.6451784e-03,  6.7220966e-04, -2.0147774e-03],
       [-2.4101972e-03, -1.9176434e-03,  2.1671599e-03, ...,
        -1.3661172e-03, -1.3818906e-03, -8.1987807e-04],
       [-1.6283204e-03, -4.0112906e-03,  2.5617480e-03, ...,
        -3.6272124e-04, -1.1886849e-04, -2.5138080e-03],
       ...,
       [-2.4189497e-03, -5.2452169e-04,  4.6270331e-03, ...,
        -7.0718667e-03, -3.3432364e-03, -1.8380929e-03],
       [ 1.5326069e-03, -1.4740487e-03,  3.8775629e-03, ...,
        -9.5384319e-05, -6.2146760e-04, -2.4011079e-03],
       [ 1.3725453e-03,  1.5336208e-03, -7.2143367e-04, ...,
         9.6710742e-04,  4.9143750e-03, -2.6023157e-03]], dtype=float32),
                    'w': array([[[-1.70202069e-02, -1.97600815e-02, -1.55001692e-02, ...,
         -8.71328916e-03,  1.83615531e-03,  9.72294621e-03],
        [ 1.58263091e-02,  6.49763329e-04,  1.71479937e-02, ...,
         -5.63265057e-03, -2.26127431e-02,  3.16370688e-02],
        [-3.67097347e-03, -1.10905785e-02,  1.21303340e-02, ...,
          1.62765887e-02,  2.08019326e-03,  8.82455520e-03],
        ...,
        [-2.60162316e-02,  2.52318997e-02,  2.38402616e-02, ...,
          1.30455270e-02, -1.14394212e-02,  3.46929883e-05],
        [ 9.73733980e-03,  4.70478972e-03,  1.08287651e-02, ...,
         -7.70605751e-04,  1.00928349e-02,  1.45578319e-02],
        [-9.30155907e-03,  1.85461044e-02, -3.05988593e-03, ...,
         -4.52069426e-03, -1.08060869e-03, -1.10367388e-02]],

[[-1.74068902e-02, -1.48470709e-02, -9.53600742e-03, ...,
         -1.10044722e-02,  8.05367716e-03,  1.61478389e-02],
        [-2.69844499e-03, -7.05788750e-03, -4.20461409e-03, ...,
          8.08789209e-03, -4.07999987e-03, -1.11394729e-02],
        [ 1.86929721e-02,  1.02935024e-02,  1.28320360e-03, ...,
          1.40923960e-02, -6.32655201e-03, -2.62563000e-03],
        ...,
        [-2.46962695e-03, -1.17938379e-02,  3.15637365e-02, ...,
          3.51362955e-03, -1.39427586e-02,  3.52351391e-03],
        [ 1.53678022e-02,  5.71628381e-03,  1.96273904e-02, ...,
          2.16926597e-02,  1.51803009e-02, -1.19672865e-02],
        [-2.41213888e-02,  1.37166046e-02, -9.67437882e-05, ...,
          1.07835382e-02, -3.45099866e-02, -6.09378098e-03]],

[[ 1.91426817e-02, -1.62930451e-02, -7.09466171e-03, ...,
         -1.36547908e-02,  6.53729355e-03,  3.19066457e-03],
        [-1.49405736e-03,  5.88373095e-03, -1.28640700e-02, ...,
          8.93671997e-03,  1.54415714e-02, -6.72838767e-04],
        [-7.46562984e-03,  3.79460258e-03,  1.07860686e-02, ...,
          6.66858535e-03,  5.15385717e-03,  5.26480796e-03],
        ...,
        [ 1.82353798e-03, -1.13049271e-02,  7.74198445e-03, ...,
         -2.71698833e-02,  1.20480824e-02,  3.63867432e-02],
        [ 2.19169483e-02,  1.53402993e-02,  1.00292396e-02, ...,
          5.99698629e-03, -1.43431472e-02,  1.56054110e-03],
        [-1.79829281e-02, -2.50810501e-03, -1.57014988e-02, ...,
          9.42519680e-03,  3.48421745e-03,  3.68633075e-03]],

...,

[[ 4.90663107e-03,  1.63754337e-02, -5.55653777e-03, ...,
          7.05450308e-03, -1.40327699e-02, -3.12101934e-02],
        [-6.11507287e-03, -6.79427292e-03,  3.43121663e-02, ...,
          3.12275952e-03,  2.41973102e-02, -1.44895550e-03],
        [-2.65017804e-03,  4.19948762e-03,  6.97611552e-03, ...,
         -2.31507071e-03, -1.65484995e-02, -2.37833951e-02],
        ...,
        [ 2.73407000e-04, -1.85697116e-02,  4.97883512e-03, ...,
         -3.34481453e-03,  3.29584517e-02, -7.54478853e-04],
        [ 5.63140726e-03, -3.34599242e-03,  1.72991473e-02, ...,
          2.52891388e-02,  1.00078480e-02, -3.95365618e-03],
        [-1.22762118e-02, -4.72204556e-04, -1.54073257e-02, ...,
          5.23721008e-03,  9.24056396e-03, -2.84674251e-03]],

[[ 1.93443075e-02, -2.30221227e-02, -2.40172129e-02, ...,
         -9.61173140e-03, -1.31262233e-03,  5.15564717e-03],
        [-1.38371121e-02,  8.29380658e-03, -1.08281514e-02, ...,
          2.63987221e-02,  1.38669107e-02,  1.14327669e-02],
        [ 1.66552216e-02, -1.05494410e-02, -1.11502726e-02, ...,
         -1.37190362e-02,  4.39156080e-03,  1.46731082e-02],
        ...,
        [-8.03474430e-03, -8.12487956e-03,  1.48500167e-02, ...,
         -2.19114274e-02,  8.74883030e-03,  2.59685051e-02],
        [-1.69046074e-02,  1.63708795e-02, -4.67114551e-05, ...,
          1.76620223e-02,  4.99137817e-03, -6.18379982e-03],
        [-4.64541418e-03, -5.41443657e-03, -2.18130322e-03, ...,
          2.25728247e-02,  4.93262382e-03, -5.16696228e-03]],

[[-4.07601148e-03,  1.09111238e-02,  1.46400845e-02, ...,
          1.44178495e-02, -1.41544109e-02, -1.25444625e-02],
        [-5.62398247e-02, -7.10166758e-04,  6.04600087e-03, ...,
          3.83043638e-03,  1.28686754e-02,  1.30163999e-02],
        [-3.78315017e-04, -8.86098854e-03, -9.61992983e-03, ...,
          2.75602285e-03,  5.36391279e-03,  5.96022047e-03],
        ...,
        [ 6.56709680e-03,  1.64176356e-02, -4.17928863e-03, ...,
         -4.09208611e-03, -4.79598151e-04, -7.82177225e-03],
        [ 2.35127136e-02, -2.28902791e-03, -5.07572992e-03, ...,
          4.67127515e-03, -7.65140029e-03, -4.28619620e-04],
        [ 2.34100297e-02,  2.30132253e-03,  3.33477068e-03, ...,
          4.12812969e-03,  9.00776312e-03,  2.72210352e-02]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.45901683, 0.51078045, 0.461296  , ..., 0.44357395,
         0.49135238, 0.52275074],
        [0.49014497, 0.50125223, 0.49712083, ..., 0.47283468,
         0.5097438 , 0.49355346],
        [0.44280928, 0.47480506, 0.39203748, ..., 0.45664167,
         0.45862848, 0.48547098],
        ...,
        [0.49133843, 0.487408  , 0.43080398, ..., 0.48800215,
         0.5086967 , 0.49353743],
        [0.45896748, 0.46875492, 0.47356227, ..., 0.48189083,
         0.46519864, 0.49883026],
        [0.46562287, 0.47814456, 0.4910004 , ..., 0.47931492,
         0.43667412, 0.4623267 ]]], dtype=float32)
                }
            }
        },
        'x_layers_15': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': array([-0.00382897, -0.01070694, -0.00421839, ..., -0.00438798,
       -0.00500148, -0.0174924 ], dtype=float32)
                    },
                    'linear': {
                        'w': array([[-0.03001062, -0.04300353,  0.03624095, ...,  0.04447658,
         0.00738414, -0.00944565],
       [ 0.04650056,  0.02834052, -0.03979881, ..., -0.00881591,
        -0.05368604, -0.01600092],
       [-0.04603452,  0.00333315,  0.02914217, ...,  0.02335356,
         0.02082835, -0.01562827],
       ...,
       [ 0.0200206 , -0.02080444, -0.05211437, ...,  0.07481808,
         0.03825745,  0.04028043],
       [ 0.00603536, -0.00296809,  0.00588007, ...,  0.01323494,
        -0.009428  ,  0.01075984],
       [-0.02157067,  0.00690726,  0.00761885, ..., -0.03510019,
         0.02382934, -0.0114434 ]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.0075785, 1.0797993, 1.0374755, ..., 1.09773  , 1.051097 ,
        1.2827269]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': array([-0.0034491 ,  0.00113755,  0.00326148, ..., -0.00202693,
       -0.00075667, -0.00020701], dtype=float32)
                    },
                    'linear': {
                        'w': array([[ 0.02785643,  0.00816482,  0.00832493, ..., -0.01484761,
        -0.01775872, -0.0023947 ],
       [-0.02799329, -0.0354571 , -0.00193841, ..., -0.04426453,
        -0.04666989,  0.0139689 ],
       [-0.03626651,  0.02228458,  0.04096157, ...,  0.02852779,
         0.02718053, -0.02925706],
       ...,
       [-0.02451389, -0.02001832,  0.01434541, ..., -0.02924969,
        -0.02220753,  0.02742435],
       [-0.01701572, -0.0317556 , -0.01858513, ..., -0.05272435,
         0.02035795, -0.03656702],
       [-0.15323555,  0.01148045,  0.00145428, ..., -0.03268117,
         0.06788057, -0.00867513]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.3310498, 1.3019437, 1.1762177, ..., 1.3272806, 1.3381895,
        1.3899907]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': array([ 0.00694113,  0.0082615 ,  0.01453089, ..., -0.00314942,
       -0.02481697, -0.00868503], dtype=float32),
                    'scale': array([ 0.00132329,  0.01813487,  0.01469828, ..., -0.01991167,
        0.01208597,  0.00790367], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': array([1.1506488, 1.1302264, 1.1266121, ..., 1.1562704, 1.0905734,
       1.1557795], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': array([[-0.00140497,  0.00156974, -0.00102715, ...,  0.00140835,
        -0.00496419, -0.00182753],
       [-0.00232494,  0.00249131, -0.0014481 , ...,  0.00208211,
        -0.00203303, -0.00034535],
       [ 0.000678  , -0.00365663,  0.00025134, ...,  0.00110168,
         0.00079804,  0.00055274],
       ...,
       [-0.00070817, -0.00151383, -0.0004914 , ..., -0.0033437 ,
        -0.00261433,  0.00016033],
       [-0.00116644, -0.00049791, -0.00098731, ..., -0.00085122,
        -0.00029645,  0.00071196],
       [-0.00285295,  0.0006996 ,  0.00339124, ..., -0.00067429,
        -0.00202245, -0.00057098]], dtype=float32),
                    'w': array([[[-1.68806985e-02,  2.91784927e-02,  1.09832585e-02, ...,
          1.80186387e-02, -1.02492515e-02,  2.72594742e-03],
        [-1.41559094e-02, -3.16049792e-02,  3.01855132e-02, ...,
         -1.00077437e-02,  1.53082535e-02, -8.09445791e-03],
        [-2.88749132e-02,  8.75951163e-03,  2.48974888e-03, ...,
         -2.28136107e-02, -3.04544698e-02,  6.84893923e-03],
        ...,
        [ 1.17152147e-02, -9.77470540e-03,  1.43085346e-02, ...,
          1.60561968e-02,  2.40543135e-03,  2.78628282e-02],
        [-1.47178555e-02, -6.04795059e-04,  2.65847845e-03, ...,
         -2.50171544e-03, -3.12985815e-02, -1.53684812e-02],
        [-1.92413330e-02,  3.39501584e-03, -1.31989354e-02, ...,
         -1.14938617e-02, -7.27869244e-03, -3.29785720e-02]],

[[-2.51294076e-02,  1.27668986e-02, -2.56550685e-03, ...,
          1.56985670e-02, -5.82664320e-03, -3.30450721e-02],
        [-6.03324771e-02, -7.38696847e-03,  2.93573202e-03, ...,
         -4.48159408e-03, -2.30578277e-02,  3.31867835e-03],
        [-1.82861313e-02,  3.01607326e-02, -4.35721353e-02, ...,
          9.26261861e-03, -1.60550953e-05, -1.68839972e-02],
        ...,
        [-1.84010547e-02, -1.00301979e-02,  3.70040201e-02, ...,
          1.11069977e-02,  3.45812812e-02, -9.35239997e-03],
        [-3.48957302e-03,  1.46794794e-02,  1.66578311e-02, ...,
         -1.54853109e-02,  9.25135706e-03,  4.91376733e-03],
        [ 3.36033688e-03,  1.26451757e-02,  7.70323025e-03, ...,
         -1.50762079e-02, -2.35536452e-02,  1.77684780e-02]],

[[ 1.14941569e-02, -9.78940167e-03,  7.67458742e-03, ...,
         -8.03827029e-03, -7.10651325e-03, -1.72787420e-02],
        [-1.41512463e-02, -2.16093659e-02,  3.72271775e-03, ...,
         -8.66229006e-04,  1.32289818e-02,  2.29452993e-03],
        [ 5.95783861e-03,  2.09717941e-03, -1.53169855e-02, ...,
          1.33602545e-02,  1.30742032e-04,  1.50898844e-02],
        ...,
        [ 4.17113490e-03, -7.91006722e-03, -6.21176790e-03, ...,
         -4.58024675e-03,  1.83616336e-02, -2.34117522e-03],
        [-1.27476146e-02, -7.38130230e-03,  1.85114555e-02, ...,
         -2.88289301e-02, -2.07957998e-03, -2.04727091e-02],
        [ 6.87939860e-03,  3.51847038e-02,  1.60446949e-02, ...,
         -2.81295255e-02, -1.03440424e-02, -2.14240467e-03]],

...,

[[-3.52899404e-03, -4.34731413e-03,  2.72429492e-02, ...,
         -1.18062599e-02,  5.22769615e-03,  6.00033579e-03],
        [-7.54508539e-04, -1.15961106e-02, -2.23721396e-02, ...,
         -2.76650377e-02, -3.82253504e-03,  8.14395398e-03],
        [ 2.81902049e-02, -5.45500498e-03, -1.26645090e-02, ...,
          1.03017306e-02,  1.95803791e-02, -2.31852345e-02],
        ...,
        [-8.75301193e-03, -6.43467065e-03, -6.80038123e-04, ...,
          1.09244101e-02,  2.55212467e-02,  1.02788834e-02],
        [-1.55728292e-02,  3.05814994e-03, -1.97694674e-02, ...,
         -1.64627731e-02,  4.28349013e-03, -1.15843862e-02],
        [-3.78981385e-05,  2.09558159e-02, -2.12039817e-02, ...,
          5.80217969e-03, -2.46106163e-02, -2.32002120e-02]],

[[-4.57244273e-03,  8.09363462e-03, -5.74351195e-03, ...,
          1.87016688e-02,  1.79824978e-02,  1.79690379e-03],
        [-1.59895532e-02,  3.20589024e-04, -1.70663465e-02, ...,
         -5.83350426e-03,  1.43780410e-02,  7.00315507e-03],
        [ 2.47828159e-02,  1.13918446e-02,  2.47797705e-02, ...,
         -1.15639449e-03,  5.61659923e-03,  1.47978831e-02],
        ...,
        [-8.78050644e-03, -1.37300082e-02,  1.32736703e-02, ...,
          2.92143179e-03, -2.22351239e-03, -1.52200460e-02],
        [-4.24316805e-03,  1.31460838e-02, -1.71210547e-03, ...,
          1.24481262e-03, -3.25176790e-02, -2.54919063e-02],
        [-1.41535541e-02, -2.04663724e-03,  2.33832579e-02, ...,
          2.86573414e-02, -6.25825021e-03, -1.24588620e-03]],

[[-1.22254500e-02,  4.47095558e-03,  1.97416022e-02, ...,
         -1.80803724e-02,  2.45548575e-03,  1.53298164e-03],
        [-2.68470813e-02, -2.59301029e-02,  1.07112490e-02, ...,
          1.96190402e-02, -3.02873785e-04, -4.77690902e-03],
        [ 6.79769693e-03,  1.80204399e-02,  1.43247014e-02, ...,
          1.25488741e-02,  8.55445396e-03, -5.46903675e-03],
        ...,
        [-1.92873869e-02, -6.78052800e-03,  1.66579057e-02, ...,
          2.67223623e-02, -4.65476373e-03,  3.95291904e-03],
        [ 1.50315964e-03, -4.17142082e-03, -3.82395945e-02, ...,
         -1.29380785e-02, -1.02337543e-02, -1.88998599e-03],
        [-3.15461121e-02,  9.29084327e-03, -1.73337702e-02, ...,
         -1.78088769e-02, -9.98180453e-03, -2.37116124e-03]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.5897061 , 0.6016743 , 0.5225944 , ..., 0.69040936,
         0.49270076, 0.60035384],
        [0.6333428 , 0.5863373 , 0.5931666 , ..., 0.58685166,
         0.5726818 , 0.6841502 ],
        [0.64522755, 0.5379591 , 0.6026368 , ..., 0.5636799 ,
         0.58634365, 0.52428657],
        ...,
        [0.5388701 , 0.55059093, 0.57001936, ..., 0.55759937,
         0.5425804 , 0.53009856],
        [0.62144923, 0.69903564, 0.78582674, ..., 0.5449928 ,
         0.83327305, 0.58033675],
        [0.5518707 , 0.51935154, 0.5387019 , ..., 0.53638905,
         0.5679283 , 0.6148828 ]]], dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': array([0.3838729 , 0.32792246, 0.36369315, 0.3325522 , 0.33127862,
       0.37433836, 0.38163787, 0.36453605, 0.32643825, 0.36529392,
       0.32667154, 0.28192905, 0.3830394 , 0.47810373, 0.3558601 ,
       0.4301477 , 0.35760036, 0.3239827 , 0.34011665, 0.35472804,
       0.36542976, 0.3011298 , 0.28342828, 0.36540616, 0.39521998,
       0.33422124, 0.3204194 , 0.40146363, 0.3714171 , 0.3723778 ,
       0.35148665, 0.35613933, 0.36487457, 0.41195023, 0.3453005 ,
       0.3637106 , 0.37228388, 0.26457918, 0.38216782, 0.30997133,
       0.4377603 , 0.4159947 , 0.40109572, 0.3988191 , 0.39574093,
       0.23585324, 0.4167122 , 0.33482814, 0.3374693 , 0.3074873 ,
       0.36557922, 0.46011403, 0.29139513, 0.3209499 , 0.3571471 ,
       0.38119125, 0.4059438 , 0.4321849 , 0.37577084, 0.2836916 ,
       0.33823603, 0.33339486, 0.35271809, 0.34291354, 0.33916453,
       0.359094  , 0.31584463, 0.40697953, 0.3733516 , 0.3247462 ,
       0.34236452, 0.2928598 , 0.354238  , 0.38166362, 0.35882866,
       0.3782471 , 0.38768637, 0.34457606, 0.41593662, 0.36847502],
      dtype=float32)
                },
                'post': {
                    'b': array([-0.00264038,  0.00694185,  0.01401373, ..., -0.00361281,
       -0.01312758, -0.00154151], dtype=float32),
                    'w': array([[[ 1.87228695e-02,  2.09279335e-03,  1.22624869e-02, ...,
         -5.55560971e-03,  4.06807842e-04,  6.86809141e-03],
        [-1.01454938e-02, -1.03525901e-02,  1.22757955e-03, ...,
          7.81850191e-04,  1.15038836e-02, -2.10440508e-03],
        [ 5.03648480e-04, -2.61391252e-02,  1.11459903e-02, ...,
          2.29138695e-02, -1.46881621e-02,  3.90026392e-03],
        ...,
        [ 1.56186810e-02,  7.08769402e-03,  8.49680137e-03, ...,
          8.11524130e-03,  1.08613428e-02,  4.19352483e-03],
        [-4.58357064e-03, -2.25086100e-02,  1.24246010e-03, ...,
         -1.20283552e-02, -1.25527671e-02,  1.78278312e-02],
        [-1.64922811e-02, -2.23112721e-02,  1.08939959e-02, ...,
          7.43396394e-03, -7.29675591e-03, -1.77015848e-02]],

[[ 4.75466391e-03, -1.92132555e-02,  3.40371463e-03, ...,
          1.33153256e-02,  2.32373960e-02,  2.16516899e-03],
        [ 1.64557528e-02,  5.59421210e-03, -1.17754447e-03, ...,
         -9.57095006e-04,  8.73026717e-03,  9.14933626e-03],
        [-5.23029407e-03, -4.41740919e-03,  1.07700741e-02, ...,
         -3.77386180e-03, -6.70594862e-03, -1.08343717e-02],
        ...,
        [ 7.89417012e-04, -2.95786746e-03,  1.58995576e-02, ...,
         -3.25451344e-02, -1.10338479e-02,  8.85534100e-03],
        [-7.09686801e-03,  6.41351659e-03,  6.62078103e-03, ...,
         -1.21601298e-02,  2.61662211e-02, -7.20187975e-03],
        [-4.33130516e-03,  1.83305852e-02, -1.25036966e-02, ...,
         -8.00358318e-03,  1.04699768e-02, -2.66333949e-02]],

[[-7.49323331e-03,  1.28021799e-02,  2.27738433e-02, ...,
         -1.21976994e-02, -1.78115778e-02,  5.02456329e-04],
        [ 1.23856612e-03, -1.61337331e-02,  7.11306790e-03, ...,
          1.88143051e-03,  3.01598758e-02, -8.00941978e-03],
        [-2.05831453e-02, -1.52782789e-02, -4.29432094e-03, ...,
          1.28702279e-02,  8.59572180e-03,  5.19281253e-03],
        ...,
        [ 1.57148868e-03, -2.37074755e-02,  1.21313985e-02, ...,
          1.04911008e-03, -1.43760431e-03, -1.47363534e-02],
        [-7.46460492e-03,  1.79007021e-03, -2.62691686e-03, ...,
          7.10256677e-03,  1.13613503e-02,  3.86804366e-03],
        [-1.83913030e-03,  5.37763909e-03,  4.04466782e-03, ...,
         -1.96334515e-02,  1.30260619e-03, -8.95215757e-03]],

...,

[[ 8.09007417e-03, -2.69561242e-02, -1.27796214e-02, ...,
          1.79399662e-02, -1.35461777e-03, -1.96673647e-02],
        [-1.67368706e-02, -6.71507325e-03, -1.38955228e-02, ...,
         -6.92330720e-03, -4.98969760e-03, -1.32145220e-02],
        [-2.74260552e-03,  1.32810650e-02,  2.57711243e-02, ...,
          1.50292385e-02,  1.73837803e-02, -1.98908206e-02],
        ...,
        [ 1.05698016e-02, -3.12781218e-03, -2.46756157e-04, ...,
          1.16313878e-03, -1.80701129e-02,  1.68538708e-02],
        [-1.28453569e-02, -4.40086462e-02, -1.30724553e-02, ...,
         -3.13281361e-03,  5.52907633e-03, -1.02039920e-02],
        [ 3.07594822e-03, -5.90648502e-03,  2.94897724e-02, ...,
          1.46030448e-02, -2.14865868e-04,  2.63953698e-04]],

[[ 2.67301127e-03, -1.09238066e-02, -5.59808977e-04, ...,
         -4.90877545e-03, -7.14305555e-03,  2.38521155e-02],
        [ 1.16410349e-02,  7.61089986e-03,  9.67153534e-03, ...,
         -6.08474109e-03,  1.44627662e-02, -6.04634406e-03],
        [ 1.29172411e-02,  1.63349509e-02, -9.34407674e-03, ...,
          2.33795983e-03,  9.74232826e-05,  2.21911855e-02],
        ...,
        [ 1.23502780e-02, -7.89832417e-03, -3.91995115e-03, ...,
          1.60134416e-02, -6.34698570e-03,  6.30159816e-03],
        [-1.69780422e-02,  2.02283189e-02,  3.53706605e-03, ...,
          1.22709125e-02, -7.23976828e-03, -1.77639965e-02],
        [-4.96006990e-03, -1.65094137e-02,  1.16877751e-02, ...,
         -8.83441977e-03,  1.87992435e-02,  1.41877811e-02]],

[[ 8.73747561e-03, -1.13629596e-02, -7.38181034e-03, ...,
          1.33837434e-02, -9.68668051e-03, -1.46931009e-02],
        [ 6.46871282e-03,  3.81157082e-03,  7.98682682e-03, ...,
         -4.13137209e-03,  1.93525141e-03, -1.31269665e-02],
        [-1.89980771e-02,  7.40556745e-04, -2.40534265e-03, ...,
          8.91357474e-03,  5.45089459e-03, -1.62918661e-02],
        ...,
        [-1.69311017e-02, -2.38751173e-02,  1.52366096e-02, ...,
          1.46889640e-02,  5.74547332e-03, -7.65995064e-05],
        [ 1.58907333e-03, -1.24421241e-02, -1.88286137e-02, ...,
          1.26177752e-02,  2.40578200e-03,  5.15837781e-03],
        [-2.36421563e-02,  2.78542517e-03,  2.30298936e-02, ...,
         -1.15507739e-02, -7.25464104e-03,  2.17000712e-02]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.5092722 , 0.47575668, 0.43138304, ..., 0.5165967 ,
         0.43076348, 0.4861471 ],
        [0.45256963, 0.49528274, 0.4453515 , ..., 0.46138558,
         0.47870448, 0.49119395],
        [0.43409872, 0.48238403, 0.47696403, ..., 0.4854055 ,
         0.47192717, 0.46046945],
        ...,
        [0.45021212, 0.48322022, 0.49625567, ..., 0.50211483,
         0.46347538, 0.4793507 ],
        [0.51061046, 0.52129245, 0.46075222, ..., 0.5144791 ,
         0.5035957 , 0.5710426 ],
        [0.45240754, 0.4492098 , 0.49739084, ..., 0.46043217,
         0.47115308, 0.5108831 ]]], dtype=float32)
                },
                'query': {
                    'b': array([[-0.00262852,  0.00193108, -0.00039411, ..., -0.00492288,
         0.00507819,  0.0045451 ],
       [-0.00436657, -0.01169078,  0.01313771, ..., -0.01245482,
         0.00836836, -0.00160828],
       [-0.00267721, -0.00322388,  0.00058989, ..., -0.00694676,
        -0.012724  , -0.00606337],
       ...,
       [ 0.00056436,  0.01125247, -0.01011393, ..., -0.00623019,
        -0.00752389,  0.00585707],
       [ 0.00551907, -0.00466406,  0.00090617, ..., -0.00812737,
        -0.00139521, -0.00883428],
       [ 0.00855688, -0.00175026, -0.01494024, ...,  0.01107276,
         0.00470326,  0.00555534]], dtype=float32),
                    'w': array([[[-2.32145227e-02, -1.80027988e-02, -1.05454139e-02, ...,
          4.62635048e-02, -3.16045135e-02,  1.83681063e-02],
        [-4.12973464e-02, -1.00914892e-02, -2.15370348e-03, ...,
          1.21688470e-02, -6.79347711e-03, -9.77568701e-03],
        [-2.93969270e-02, -5.16810594e-03, -3.31422426e-02, ...,
          4.39999218e-04, -2.24977881e-02,  1.34276571e-02],
        ...,
        [ 1.30915800e-02,  2.56901397e-03,  1.60468332e-02, ...,
          5.56441862e-03,  2.95666195e-02,  1.05255062e-03],
        [-2.43136138e-02, -1.78905595e-02,  3.00597725e-03, ...,
          1.19134772e-03,  6.64961059e-03, -1.39747770e-03],
        [-1.76110156e-02, -1.31995883e-03, -6.59585185e-03, ...,
          5.34036756e-03,  1.04377223e-02,  1.93941332e-02]],

[[-1.59265716e-02,  4.59200190e-03,  2.73639187e-02, ...,
         -1.95614416e-02, -4.63957287e-04, -1.52134933e-02],
        [ 3.00110411e-02,  3.81806716e-02, -1.72670558e-02, ...,
         -2.86256634e-02, -7.68929953e-03,  1.20591959e-02],
        [-2.01961864e-02, -5.79226250e-03,  1.04395663e-02, ...,
          1.37882191e-03, -2.57616118e-03,  2.75512505e-02],
        ...,
        [-1.45454621e-02,  4.63097473e-04,  2.53355540e-02, ...,
          5.15399985e-02,  1.92504749e-02,  5.62958085e-05],
        [ 3.63689773e-02,  2.00298103e-03, -3.90452007e-03, ...,
          3.03020328e-02,  1.40434392e-02,  1.84588656e-02],
        [ 5.03220363e-03, -1.46125546e-02, -8.35905690e-03, ...,
          9.29564983e-03, -8.23873933e-03, -1.46203302e-02]],

[[-2.96961926e-02,  3.31357419e-02,  1.84401665e-02, ...,
          3.23686898e-02, -2.69778874e-02, -7.76608149e-03],
        [-1.62376538e-02, -9.50678624e-03,  9.79299378e-03, ...,
          4.29787487e-03,  3.77430185e-03, -1.99180818e-03],
        [-1.30321151e-02,  1.70720536e-02, -1.77414063e-02, ...,
          1.33117791e-02,  2.40523904e-03,  1.37761291e-02],
        ...,
        [-1.84978321e-02,  3.02371327e-02,  2.71201041e-03, ...,
         -2.82045528e-02, -1.68058705e-02,  3.45874429e-02],
        [-4.94443206e-03, -5.50820604e-02, -2.65754614e-04, ...,
         -6.29409822e-03,  7.06172315e-03,  1.15319565e-02],
        [-4.52144071e-03,  8.39048531e-03, -2.60597691e-02, ...,
         -3.45759327e-04, -2.11154632e-02,  1.51320761e-02]],

...,

[[ 2.01858897e-02, -3.58779877e-02, -2.67656874e-02, ...,
          2.93407422e-02, -5.48194023e-03,  2.41589034e-03],
        [ 2.00146567e-02, -1.81997661e-02, -3.00319176e-02, ...,
          1.09957075e-02,  2.70355400e-02,  1.90973794e-03],
        [ 7.07499869e-03,  4.36401106e-02, -2.51662396e-02, ...,
         -2.09003929e-02,  2.06946842e-02,  8.12593102e-03],
        ...,
        [-3.48328762e-02, -1.38646103e-02,  6.14487985e-03, ...,
          6.64387504e-03,  3.75910290e-02, -2.87000574e-02],
        [-8.16619918e-02,  1.96462125e-02, -5.83259463e-02, ...,
         -2.18233112e-02,  1.53396502e-02,  7.55190253e-02],
        [ 8.62188288e-04,  1.36273121e-02,  5.71869835e-02, ...,
         -4.30222489e-02, -4.29713679e-03,  2.09438062e-04]],

[[ 2.93164281e-03, -1.39209274e-02,  1.62183400e-02, ...,
          2.56824680e-02,  8.90209991e-03, -1.58167556e-02],
        [ 4.20132000e-03,  2.76094005e-02, -1.38121424e-02, ...,
         -5.64199351e-02,  2.73115672e-02,  2.98125781e-02],
        [ 9.11192317e-03, -1.86472759e-03,  6.38794480e-03, ...,
          2.23717429e-02,  2.33593248e-02,  1.58703569e-02],
        ...,
        [-1.42117059e-02, -5.73036857e-02, -2.96167489e-02, ...,
          1.19073186e-02, -9.14105866e-03,  4.06445190e-02],
        [-2.38571335e-02, -4.01985049e-02,  3.94393094e-02, ...,
          3.32504846e-02,  5.37406327e-03,  1.87008139e-02],
        [-6.64491300e-03, -1.12482915e-02,  1.46715986e-02, ...,
         -2.59941891e-02,  1.45181650e-02,  2.81002326e-03]],

[[-8.00139830e-03,  1.62869226e-02, -2.50888988e-02, ...,
          1.60501990e-02,  1.23645673e-02,  4.42911731e-03],
        [-1.40104657e-02,  5.45975715e-02,  2.15243883e-02, ...,
         -2.07017269e-02, -5.22241881e-03, -1.60436239e-02],
        [ 1.52988229e-02,  4.63366173e-02,  2.69267075e-02, ...,
         -7.67846312e-03,  1.13997133e-02,  5.20083569e-02],
        ...,
        [ 4.15044986e-02,  6.75549870e-03,  5.89986425e-03, ...,
          3.34248096e-02, -2.61541680e-02, -8.67828447e-03],
        [-4.17082123e-02,  1.01776039e-02, -4.68776189e-02, ...,
          1.82378702e-02,  3.92180420e-02, -6.68293834e-02],
        [-5.26580913e-03,  3.55641334e-03, -4.06127376e-03, ...,
          8.88812472e-04,  4.26395908e-02,  3.76701988e-02]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.74114025, 0.6959678 , 0.8067558 , ..., 0.7407045 ,
         0.82334256, 0.7050059 ],
        [0.6365441 , 0.72303927, 0.8372024 , ..., 0.86499614,
         0.61541796, 0.63353163],
        [0.64427227, 0.76940787, 0.7169134 , ..., 0.73980546,
         0.7254454 , 0.8275267 ],
        ...,
        [0.92033076, 0.8940087 , 0.94009846, ..., 0.9679121 ,
         0.9750414 , 1.0121049 ],
        [1.0379242 , 0.98727036, 1.0911665 , ..., 1.0971216 ,
         0.8876684 , 1.1740981 ],
        [0.8464151 , 0.7297731 , 0.7977142 , ..., 0.81558186,
         0.8316571 , 0.6909688 ]]], dtype=float32)
                },
                'value': {
                    'b': array([[ 4.3092333e-03, -1.0670313e-03,  1.0951947e-03, ...,
        -7.5847393e-06, -2.9846833e-03, -2.8228675e-05],
       [ 1.0933492e-03,  3.7175193e-03, -1.4860635e-03, ...,
         1.9951609e-03, -3.4172609e-03,  4.1828798e-03],
       [ 1.5554003e-03, -3.2844695e-03, -4.2517087e-03, ...,
        -6.4385339e-04, -2.6835120e-04, -5.1554147e-04],
       ...,
       [-5.4495255e-03, -4.6499642e-03, -3.1639240e-03, ...,
        -2.4868713e-03, -5.6621089e-04, -2.1520587e-05],
       [-1.2494889e-03,  4.1477205e-03, -2.2556106e-04, ...,
        -4.7293715e-03, -1.6261863e-03, -1.2881723e-03],
       [ 4.9109582e-04, -1.3274476e-03,  1.4091528e-03, ...,
        -1.3600906e-03,  5.9296563e-03,  8.0530928e-04]], dtype=float32),
                    'w': array([[[-2.69760992e-02,  1.25788096e-02, -7.61376414e-03, ...,
         -1.52490120e-02,  5.33550233e-03,  7.93699082e-03],
        [ 1.05836606e-02,  1.82690602e-02,  1.03190932e-02, ...,
         -1.38023579e-02,  1.13194250e-02, -1.86773445e-02],
        [ 6.18274929e-03,  2.25341134e-02, -2.55283359e-02, ...,
          9.22846049e-03,  2.76181158e-02, -8.00884049e-03],
        ...,
        [-2.94117315e-04, -3.88901588e-03,  1.62602384e-02, ...,
          6.27086684e-03, -2.42737625e-02,  1.65726133e-02],
        [-2.14569233e-02,  4.66705300e-03,  7.93393422e-03, ...,
          2.82598324e-02,  1.51329385e-02, -9.63309221e-03],
        [ 1.53947799e-02, -1.65367703e-04,  2.37261318e-03, ...,
          5.47492637e-05,  1.14596752e-03, -3.79634369e-03]],

[[-2.12856811e-02,  2.55563576e-02, -2.29895581e-02, ...,
         -4.12435234e-02,  2.32004724e-03, -2.19861474e-02],
        [ 4.69826860e-03, -8.32690205e-03, -1.21832453e-03, ...,
          4.53032879e-03, -6.34168973e-03, -6.76035474e-04],
        [-9.90476273e-03,  1.78956557e-02,  2.68151294e-02, ...,
          2.08922848e-03, -1.61815435e-02,  6.45424612e-03],
        ...,
        [ 2.88140569e-02,  2.09164470e-02, -3.65769416e-02, ...,
          8.46528728e-03,  1.02296118e-02, -5.86308585e-03],
        [ 2.98043666e-03, -1.08131971e-02,  7.12977024e-03, ...,
         -8.64494033e-03, -1.87955443e-02, -1.40109574e-02],
        [ 1.14591625e-02, -4.57976758e-03,  1.01466877e-02, ...,
          3.96450702e-03, -2.72908900e-02,  2.38437783e-02]],

[[-1.19842635e-03, -9.33958590e-03, -1.45958671e-02, ...,
          2.43892260e-02,  6.74558990e-03,  8.69211089e-03],
        [-8.42524506e-03,  1.27714558e-03,  2.84071686e-03, ...,
          1.88047681e-02, -9.36965458e-03,  4.38218354e-04],
        [ 1.51561005e-02,  1.56216323e-02,  1.05084777e-02, ...,
          1.30412793e-02, -8.16613343e-03,  6.35856250e-03],
        ...,
        [ 1.68796221e-03,  1.24581382e-02, -2.13506520e-02, ...,
          4.05836059e-03, -1.78233185e-03,  1.23743676e-02],
        [ 2.01378949e-02,  2.26847921e-02, -3.99035029e-03, ...,
          7.28959218e-03, -1.44184474e-02,  1.14310374e-02],
        [ 4.34358558e-03, -1.59233098e-03, -1.23905158e-02, ...,
          1.13441162e-02,  1.13115525e-02,  7.27602467e-03]],

...,

[[-2.58948598e-02,  3.23837996e-03, -5.11970138e-03, ...,
         -1.05177406e-02,  1.30696967e-02,  3.87883186e-02],
        [-1.37563795e-02, -9.89757758e-03, -3.23323812e-03, ...,
         -2.73726531e-03,  5.45451359e-04, -4.80779260e-03],
        [-3.55882896e-03, -1.21648167e-03, -1.36092743e-02, ...,
          2.66434904e-03,  9.59650520e-03,  1.95894130e-02],
        ...,
        [-8.75643641e-03, -3.09043634e-03, -8.64144694e-03, ...,
         -1.50798401e-02,  3.44238691e-02,  1.57216787e-02],
        [ 2.49033216e-02,  1.09714307e-02, -3.79896082e-04, ...,
         -8.45094398e-03, -5.89205744e-03, -1.01101240e-02],
        [-6.73529971e-03, -1.21161053e-02,  1.67450029e-02, ...,
         -1.84907764e-02, -8.80714878e-03,  8.20393860e-03]],

[[-1.12870345e-02,  1.09994300e-02, -1.06317066e-02, ...,
          4.31686174e-03,  4.80118766e-03,  4.04219702e-03],
        [ 7.68832536e-03, -2.63462011e-02,  1.88501691e-03, ...,
          1.50466273e-02,  4.11725231e-03,  1.43704433e-02],
        [ 1.34616287e-03, -1.88602377e-02,  1.17581086e-02, ...,
         -6.29825192e-03,  4.59586922e-03, -6.65642740e-03],
        ...,
        [ 7.73213618e-03, -9.69373062e-03,  2.04752316e-03, ...,
         -7.54346699e-03, -2.49041477e-03, -1.36581650e-02],
        [ 2.06210203e-02, -2.00720616e-02, -6.40665507e-03, ...,
         -1.20953182e-02,  9.00195353e-03,  1.10645602e-02],
        [-9.00809560e-03, -5.40262414e-03,  7.99079426e-03, ...,
         -1.92610535e-03,  1.20146759e-02, -1.94568764e-02]],

[[ 1.12230079e-02,  4.17726627e-03, -1.50599014e-02, ...,
         -1.54130748e-02,  4.49147914e-03,  2.02064924e-02],
        [ 7.14984071e-03, -1.13470023e-02, -7.55088637e-03, ...,
          2.19306722e-03, -5.27518662e-03,  1.26894163e-02],
        [ 1.88901164e-02, -2.14834549e-02, -5.00168302e-04, ...,
         -1.98321044e-02,  2.19250051e-03,  1.37647400e-02],
        ...,
        [ 8.68065562e-03,  1.22577073e-02, -1.24850739e-02, ...,
         -7.33076176e-03, -2.36104825e-03,  6.77861832e-03],
        [ 1.20055564e-02, -7.20065180e-03,  1.14093032e-02, ...,
         -1.32740131e-02, -2.68567689e-02, -3.12830806e-02],
        [ 8.80962703e-03, -2.94224042e-02, -1.45735533e-03, ...,
         -2.19043531e-02,  1.76994188e-04,  3.11435293e-03]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.5286878 , 0.45882484, 0.4361065 , ..., 0.5056554 ,
         0.44143242, 0.48562354],
        [0.4235816 , 0.4953367 , 0.42249233, ..., 0.4500553 ,
         0.47782612, 0.4843576 ],
        [0.45751032, 0.46943435, 0.49769756, ..., 0.47508574,
         0.4279509 , 0.4632025 ],
        ...,
        [0.43921453, 0.46600172, 0.49223578, ..., 0.49168015,
         0.4771468 , 0.4779434 ],
        [0.5188756 , 0.5586255 , 0.4437881 , ..., 0.5302013 ,
         0.53594166, 0.62284106],
        [0.4402943 , 0.44598532, 0.51921946, ..., 0.47070628,
         0.4903309 , 0.5116028 ]]], dtype=float32)
                }
            }
        },
        'x_layers_16': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': array([-0.01093779, -0.006601  , -0.01157581, ..., -0.00636036,
        0.00338994, -0.00561759], dtype=float32)
                    },
                    'linear': {
                        'w': array([[-0.07065611,  0.0366316 ,  0.00286838, ...,  0.04645369,
        -0.00233936, -0.00317006],
       [ 0.01336469,  0.00203272, -0.04045084, ...,  0.00291746,
        -0.02290988, -0.0063614 ],
       [-0.00353894,  0.0236262 , -0.0291876 , ..., -0.01251738,
        -0.01119486,  0.0026152 ],
       ...,
       [-0.00828904,  0.0029501 ,  0.00390786, ...,  0.02516262,
        -0.01356525,  0.00672958],
       [-0.00160194, -0.02658143,  0.05103948, ...,  0.01940908,
        -0.03386628,  0.02277668],
       [-0.00529792,  0.04206442,  0.01578536, ..., -0.01186924,
         0.02141595,  0.01679806]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.1466464, 1.076408 , 1.1396934, ..., 1.1958363, 0.9458777,
        1.028311 ]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': array([-0.00202641,  0.00375642,  0.00326318, ..., -0.00049565,
       -0.00301311,  0.00015281], dtype=float32)
                    },
                    'linear': {
                        'w': array([[-0.00584749, -0.01089616, -0.05799555, ..., -0.03713508,
         0.00729555, -0.01389926],
       [-0.04660178, -0.027352  , -0.04065514, ...,  0.03577564,
        -0.03868281,  0.00700549],
       [-0.03919972, -0.01063161,  0.0586987 , ...,  0.01688408,
         0.04164346, -0.01884241],
       ...,
       [ 0.02697367, -0.01475049,  0.04031842, ..., -0.04789248,
        -0.02740616,  0.0376101 ],
       [-0.03518107,  0.0076696 ,  0.02464686, ..., -0.01843506,
         0.00698606, -0.04271198],
       [ 0.05981452,  0.03818025, -0.0107589 , ..., -0.01857778,
        -0.00216738,  0.00547846]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.2693958, 1.3006517, 1.2394711, ..., 1.3128855, 1.2633555,
        1.2886195]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': array([ 0.00151769,  0.00547054,  0.01455967, ..., -0.0095241 ,
       -0.02066464, -0.00408657], dtype=float32),
                    'scale': array([ 0.00557752, -0.0064744 ,  0.02385767, ..., -0.0015815 ,
        0.0141864 ,  0.02011438], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': array([1.1342393, 1.0935222, 1.1052631, ..., 1.1164135, 1.0588753,
       1.131423 ], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': array([[-2.5851612e-03,  1.8234068e-03,  1.4820995e-03, ...,
        -1.1297370e-03, -1.0774004e-03, -8.2510058e-05],
       [-1.3637317e-03, -1.5217112e-03, -2.3586715e-04, ...,
         7.5874908e-04,  1.4486602e-03,  1.9991904e-04],
       [-1.3591145e-03, -8.1638672e-04, -3.9848150e-03, ...,
        -2.2466586e-03, -9.1814104e-04,  5.3033553e-04],
       ...,
       [ 4.4141963e-04,  1.4761785e-03,  3.0662501e-04, ...,
         1.2143034e-03, -4.2294950e-04, -3.1919666e-03],
       [-3.9186152e-03,  1.6968115e-03, -2.6914002e-03, ...,
        -2.7568685e-03, -4.9815909e-03, -5.5790623e-04],
       [ 5.4388520e-06,  1.6627987e-03,  1.9495675e-03, ...,
        -1.2228332e-03,  3.4314577e-04, -1.3438117e-03]], dtype=float32),
                    'w': array([[[ 1.42219989e-02,  2.71342508e-03,  1.42611526e-02, ...,
         -6.63716695e-04,  2.06843554e-03, -1.18770655e-02],
        [-1.26144430e-02, -2.13406812e-02,  1.53633235e-02, ...,
         -2.08577700e-02,  5.63759264e-03,  3.21701840e-02],
        [ 9.53059178e-03, -3.01317032e-02,  2.32374743e-02, ...,
         -4.51727508e-04, -9.19227581e-03, -3.94389872e-03],
        ...,
        [-2.69971937e-02, -6.48251968e-03,  4.16581379e-03, ...,
          7.08622346e-03,  2.02608900e-03, -1.23441136e-02],
        [-1.16614746e-02,  2.02821363e-02, -9.51786153e-03, ...,
          1.27844163e-03,  3.91502958e-03, -2.33424399e-02],
        [-1.08019337e-02, -1.44250272e-02, -1.16862142e-02, ...,
          1.82441156e-02, -1.74487494e-02,  2.79446272e-03]],

[[ 2.53047375e-03,  2.84810401e-02,  1.25802467e-02, ...,
          2.56162640e-02, -1.31336949e-03, -1.94497891e-02],
        [-1.80130005e-02, -7.90979713e-04,  4.87560453e-03, ...,
         -3.43334824e-02,  8.54055490e-03, -8.87987204e-03],
        [ 1.11784972e-02,  9.10202786e-03, -2.71228198e-02, ...,
          7.27072777e-03, -6.56833127e-03, -1.01706078e-02],
        ...,
        [ 1.02983648e-02, -1.29664764e-02, -8.20261519e-03, ...,
         -3.73837054e-02, -1.43253710e-02,  2.38913670e-03],
        [ 1.23405742e-04,  1.45100430e-02, -1.04996131e-03, ...,
          1.66919781e-03, -1.19626219e-03,  1.71232875e-03],
        [-1.37245487e-02,  1.91375371e-02,  1.07173482e-02, ...,
          1.22118117e-02,  8.37213732e-03, -1.25802523e-02]],

[[ 3.01345764e-03, -3.56052606e-03, -6.56895898e-03, ...,
         -3.23878117e-02,  1.46417497e-02,  2.68466435e-02],
        [ 1.86228529e-02, -1.93614105e-05,  2.39351522e-02, ...,
         -1.78275555e-02, -3.13101627e-04, -1.01873241e-02],
        [ 9.76173393e-03, -1.63431577e-02, -5.70028182e-03, ...,
          7.12055014e-03,  2.94796284e-03,  1.08349901e-02],
        ...,
        [-1.65624935e-02, -1.13580013e-02,  2.09414251e-02, ...,
          2.59363241e-02, -9.99328564e-04, -4.62058792e-03],
        [-1.66377809e-03, -2.15011952e-03,  2.37581246e-02, ...,
          1.81582570e-02,  1.82578003e-03, -1.20056551e-02],
        [-1.84043348e-02,  1.23392139e-02,  9.54618584e-03, ...,
          1.27128875e-02, -2.06954218e-03,  7.30694924e-03]],

...,

[[ 4.36866749e-03,  1.02152592e-02,  1.94614343e-02, ...,
         -2.70075835e-02, -1.58981408e-03,  1.20746652e-02],
        [-5.02618076e-03,  2.33588312e-02, -6.85508503e-03, ...,
          9.76660755e-03, -2.05430258e-02,  6.67601312e-03],
        [-6.18096534e-03, -1.22290226e-02, -3.62144522e-02, ...,
         -1.36084866e-03, -6.55003125e-03,  1.55497603e-02],
        ...,
        [-2.68481914e-02, -2.72951983e-02, -3.29536007e-04, ...,
         -3.30555514e-02, -2.31367815e-03,  1.11331455e-02],
        [ 2.81314063e-03, -6.46386202e-03,  6.30827574e-03, ...,
         -3.65220606e-02,  1.41527597e-02,  2.38706060e-02],
        [-3.54880467e-02,  1.05052870e-02, -5.94132068e-03, ...,
          2.51334570e-02,  1.57475118e-02, -1.47424713e-02]],

[[-1.43188508e-02,  6.28981553e-03,  1.01756398e-02, ...,
          6.98139658e-03,  2.00312771e-02, -1.34765673e-02],
        [ 1.69506837e-02,  2.58275599e-04, -1.97218768e-02, ...,
          6.87824003e-03,  2.01173732e-03,  2.02499498e-02],
        [ 4.77242889e-03,  1.42306611e-02, -3.13527789e-03, ...,
         -2.58080149e-03,  1.69282984e-02, -3.87123949e-03],
        ...,
        [-2.21410813e-03,  1.15614245e-02,  1.13762384e-02, ...,
          5.17080817e-03, -2.26386152e-02, -9.88213718e-03],
        [-7.42880255e-03, -1.30429752e-02,  8.99086241e-03, ...,
         -1.40716257e-02, -1.12629952e-02,  3.39667057e-03],
        [ 2.30226722e-02,  2.37231317e-04,  5.57143241e-03, ...,
         -4.23729280e-03, -2.89159808e-02, -8.84004030e-03]],

[[-6.31908793e-03,  6.94034388e-03,  7.63501693e-03, ...,
         -1.41769815e-02, -6.49016025e-03, -6.64137816e-03],
        [ 2.29326990e-02, -1.21046333e-02, -2.68131159e-02, ...,
         -1.35539537e-02, -3.14271003e-02,  2.52824724e-02],
        [-6.18695933e-03,  4.36464651e-03, -2.61276700e-02, ...,
         -1.36762187e-02, -7.50785926e-03,  4.96061035e-02],
        ...,
        [ 2.64439005e-02,  8.17085337e-03, -2.24624947e-03, ...,
         -4.54828292e-02,  5.85463829e-03,  1.52172390e-02],
        [-4.02525906e-03, -4.76133544e-03,  1.62002854e-02, ...,
          4.21542348e-03,  2.11307444e-02, -2.67693736e-02],
        [-1.22092944e-02,  1.79102104e-02, -1.53264159e-03, ...,
          1.92070529e-02, -1.04104066e-02,  6.43276563e-03]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.4371684 , 0.4283419 , 0.4048263 , ..., 0.61348206,
         0.527551  , 0.5657912 ],
        [0.67347556, 0.79177916, 0.576183  , ..., 0.66022766,
         0.54075885, 0.68674254],
        [0.49249986, 0.4732276 , 0.5165178 , ..., 0.4879495 ,
         0.45629343, 0.56334   ],
        ...,
        [0.56663483, 0.50988877, 0.4879624 , ..., 0.5967907 ,
         0.46159732, 0.37048253],
        [0.47107694, 0.48555848, 0.6218902 , ..., 0.55186844,
         0.48307857, 0.48110196],
        [0.59785193, 0.53165376, 0.5089456 , ..., 0.51613504,
         0.61920094, 0.4986022 ]]], dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': array([0.34839326, 0.34487322, 0.30048433, 0.40271607, 0.36474398,
       0.37258643, 0.33030462, 0.32204038, 0.2811675 , 0.3941069 ,
       0.40174028, 0.31793132, 0.34787026, 0.35276464, 0.36014393,
       0.38798308, 0.36074895, 0.36447507, 0.35500136, 0.38915566,
       0.38850445, 0.37506366, 0.3692204 , 0.37950742, 0.39861467,
       0.387693  , 0.40703672, 0.35293734, 0.40224758, 0.34825236,
       0.3776654 , 0.39786416, 0.34274727, 0.34450045, 0.45802835,
       0.3469762 , 0.40184072, 0.3448745 , 0.44639596, 0.37119558,
       0.3595001 , 0.34055847, 0.34886846, 0.32589296, 0.4076769 ,
       0.2716943 , 0.36786398, 0.3192062 , 0.401973  , 0.3555044 ,
       0.4049436 , 0.36825705, 0.3393343 , 0.3791553 , 0.39327073,
       0.37439272, 0.37110728, 0.28624743, 0.36308977, 0.37162504,
       0.35790285, 0.3320287 , 0.34310248, 0.3696956 , 0.36224914,
       0.36394373, 0.3157535 , 0.3760167 , 0.38727936, 0.3700242 ,
       0.35368103, 0.39586368, 0.31667173, 0.32127523, 0.41776994,
       0.35321888, 0.35742685, 0.33300877, 0.39574778, 0.36631098],
      dtype=float32)
                },
                'post': {
                    'b': array([-0.00294997,  0.00812847,  0.0093169 , ..., -0.0007974 ,
       -0.01477665, -0.0010464 ], dtype=float32),
                    'w': array([[[-6.26317598e-03, -4.82488278e-04, -8.57751351e-03, ...,
         -2.05452647e-02,  2.42175907e-02, -6.70188665e-03],
        [ 1.41657665e-02, -6.73066778e-03, -1.66489817e-02, ...,
         -1.62274856e-02,  2.53450533e-04, -9.73656075e-04],
        [ 1.20002348e-02,  5.99002978e-03, -2.94627491e-02, ...,
         -3.66344047e-03, -1.12276413e-02,  2.94008339e-03],
        ...,
        [-7.90430978e-03,  8.99317674e-03, -1.75724679e-03, ...,
         -8.32430273e-03,  2.25645519e-04,  5.43238735e-03],
        [-1.56619363e-02, -8.48766067e-04, -1.67979728e-02, ...,
          7.18162162e-03, -7.05741858e-03,  1.16398856e-02],
        [-1.49521208e-03, -9.60188918e-03, -1.99567042e-02, ...,
         -8.80062301e-03,  1.89761408e-02,  7.87581783e-03]],

[[ 1.62793212e-02,  1.58010870e-02, -1.25539757e-03, ...,
          4.27472260e-04,  2.28527058e-02,  1.85773224e-02],
        [ 1.15112250e-03, -2.62202206e-03,  1.37910089e-02, ...,
         -3.74394516e-03,  1.05342958e-02, -7.06663262e-03],
        [ 2.02836245e-02,  9.12415516e-03, -1.51518239e-02, ...,
         -3.56140025e-02, -5.81971975e-03,  1.59042282e-03],
        ...,
        [-4.57512494e-03, -1.46214841e-02,  2.32926253e-02, ...,
         -3.52016799e-02, -1.27856163e-02,  2.35062912e-02],
        [ 2.49866433e-02,  9.26466659e-03,  9.99525469e-03, ...,
          2.11897003e-03,  6.58398308e-03, -5.24292514e-03],
        [-8.00249819e-03, -2.04931665e-02,  3.33766895e-03, ...,
          2.70243548e-03, -3.64281051e-03, -3.01628723e-03]],

[[-2.16035434e-04,  1.60552654e-02, -6.09985180e-03, ...,
         -1.69639103e-02,  3.61900665e-02,  8.08812957e-03],
        [-8.98846798e-03,  1.25682987e-02, -7.86608178e-03, ...,
         -1.26995910e-02,  2.57981978e-02, -1.25881545e-02],
        [ 2.47844704e-03, -4.85568028e-03, -1.58151332e-02, ...,
          1.77433044e-02,  3.55217569e-02,  1.23261549e-02],
        ...,
        [ 3.91385844e-03,  5.75136859e-03, -1.87795260e-03, ...,
         -1.53975294e-03,  2.74369004e-03,  3.85678117e-03],
        [ 6.07362064e-03, -3.77777331e-02,  3.22198421e-02, ...,
         -1.55727530e-03, -5.60227549e-03,  1.19092884e-02],
        [-1.37926480e-02, -2.15964504e-02, -8.86894856e-03, ...,
          4.34755953e-03, -6.00206014e-03, -4.58724936e-03]],

...,

[[ 1.53349421e-03,  6.45877517e-05, -7.65935890e-03, ...,
          7.20648700e-03,  1.17701357e-02,  5.33646997e-03],
        [-2.92145554e-03, -1.15781766e-03,  6.75866148e-04, ...,
          1.55856684e-02, -2.63088732e-04,  1.26915295e-02],
        [-3.74787441e-03, -3.84216895e-03,  1.06002167e-02, ...,
          1.74155980e-02,  1.23982225e-02, -5.12109650e-03],
        ...,
        [-4.73917881e-03,  1.47920642e-02,  6.57978328e-03, ...,
         -1.97115690e-02,  6.83281280e-04,  5.95720718e-03],
        [-4.85525560e-03,  1.17602162e-02, -6.85848808e-03, ...,
          5.27203549e-04, -8.45956896e-03, -3.33079286e-02],
        [-5.34882396e-03,  9.10879392e-03, -1.79529581e-02, ...,
          6.90989941e-03, -2.76847347e-03,  5.28000854e-03]],

[[-4.04089736e-03, -6.92601362e-03, -1.22537529e-02, ...,
         -3.34773958e-02,  1.81635804e-02,  3.22751730e-04],
        [-3.20629105e-02, -6.58302614e-03,  1.21787228e-02, ...,
          1.41442996e-02,  8.32308840e-04, -1.96934049e-03],
        [ 1.90588888e-02,  7.19227130e-03,  6.49797451e-03, ...,
         -8.30287021e-03,  7.67773530e-03, -1.27140507e-02],
        ...,
        [-6.85527129e-03, -5.95056452e-04, -1.26059903e-02, ...,
         -1.26679577e-02,  2.13158205e-02,  7.24648125e-03],
        [-7.20444586e-05, -1.07246032e-03,  1.99966449e-02, ...,
          1.51923774e-02,  2.19440497e-02,  4.96581383e-03],
        [ 1.85496174e-02, -1.31261023e-02, -8.59257579e-03, ...,
         -4.57279310e-02,  1.83265395e-02,  2.07161880e-03]],

[[-1.23030320e-02,  2.26234575e-03,  1.09223975e-03, ...,
          3.52696120e-03,  1.31098898e-02, -3.36450264e-02],
        [ 7.17036705e-03, -4.04133112e-04,  1.35272148e-03, ...,
          9.51828528e-03,  4.72732633e-03, -9.87181440e-03],
        [-3.73343774e-03, -6.48180163e-03, -1.70778241e-02, ...,
         -6.57120021e-03,  2.25233845e-02, -3.19988653e-03],
        ...,
        [ 1.78108849e-02,  2.42996644e-02, -1.22569716e-02, ...,
         -3.13344225e-02, -1.28056565e-02, -6.51384518e-03],
        [ 2.63769217e-02, -2.26650685e-02, -8.46231449e-03, ...,
         -9.00840573e-03,  2.37338874e-03,  1.78988148e-02],
        [ 2.86986656e-03, -1.65359688e-03,  2.27783080e-02, ...,
         -4.38299682e-03,  2.24830187e-03, -3.57659650e-04]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.3777828 , 0.3670186 , 0.49362043, ..., 0.4880705 ,
         0.5159647 , 0.44435748],
        [0.45241416, 0.5037893 , 0.52024466, ..., 0.55471885,
         0.50103605, 0.51148343],
        [0.42501944, 0.49511734, 0.48646465, ..., 0.40938014,
         0.47122234, 0.46864575],
        ...,
        [0.41736284, 0.42159742, 0.3286683 , ..., 0.43382737,
         0.4866446 , 0.46668902],
        [0.5006488 , 0.5211803 , 0.50775874, ..., 0.36017203,
         0.5024318 , 0.4585985 ],
        [0.45917118, 0.48158947, 0.46237472, ..., 0.4969658 ,
         0.52393556, 0.45787004]]], dtype=float32)
                },
                'query': {
                    'b': array([[-0.01165662,  0.02085786,  0.02257565, ...,  0.00330295,
        -0.00433043, -0.00432216],
       [ 0.0011895 ,  0.00280689, -0.00527828, ..., -0.00170289,
         0.00037589, -0.00220599],
       [-0.00748594, -0.01677281, -0.00804976, ..., -0.01231432,
        -0.01559735,  0.01375802],
       ...,
       [-0.01084539, -0.00508603,  0.01091589, ..., -0.00500977,
        -0.01142354,  0.01658001],
       [-0.00247246,  0.0003816 ,  0.00308422, ..., -0.00597533,
        -0.00084152, -0.0063736 ],
       [ 0.00854591, -0.01875939, -0.01584159, ...,  0.01792025,
        -0.00124592,  0.01076551]], dtype=float32),
                    'w': array([[[-3.76275554e-02, -7.15244235e-03,  3.47784422e-02, ...,
          4.98246141e-02, -4.11462523e-02, -1.32022621e-02],
        [-2.34063361e-02, -1.19711217e-02, -2.04937831e-02, ...,
          1.11456187e-02,  2.34308112e-02, -2.43430343e-02],
        [-9.23752226e-03,  1.40757095e-02, -1.59042459e-02, ...,
          1.05783297e-03, -1.82259232e-02, -8.06174055e-03],
        ...,
        [ 1.27868764e-02,  1.28190068e-03, -4.98180129e-02, ...,
         -4.39898856e-02,  6.19128020e-03,  6.68785395e-03],
        [-5.87123586e-03, -2.46423464e-02,  2.46517453e-02, ...,
          2.01327037e-02, -3.63838188e-02,  1.02107450e-02],
        [-1.71772437e-03,  6.64134882e-03, -1.76312551e-02, ...,
          5.36030717e-03, -1.41256815e-03, -1.02146827e-02]],

[[ 1.95211684e-03,  3.85124795e-03,  4.45124842e-02, ...,
         -3.77109535e-02,  9.09731444e-03,  3.92662324e-02],
        [-5.51166479e-03, -3.34020182e-02,  1.94003973e-02, ...,
          3.36082242e-02, -1.37527669e-02, -1.78822991e-03],
        [-2.58822292e-02,  1.40753314e-02,  1.96352657e-02, ...,
         -5.62249590e-03, -1.51821468e-02,  2.34131534e-02],
        ...,
        [ 6.23034220e-03, -2.51418306e-03, -2.09691022e-02, ...,
         -3.90223153e-02,  1.31893782e-02,  4.52255830e-02],
        [ 2.66062096e-02, -2.06454862e-02, -1.68834071e-04, ...,
         -2.01831269e-03, -7.25879939e-03, -6.19313419e-02],
        [ 9.77389328e-03,  5.33414353e-03, -2.87315268e-02, ...,
          3.20306867e-02,  3.47274169e-03, -2.70120241e-02]],

[[-2.32200809e-02,  3.44704241e-02,  5.81018999e-02, ...,
          1.72551721e-02,  3.72326709e-02, -2.06885207e-02],
        [-4.41761836e-02,  1.58804487e-02, -2.10596472e-02, ...,
          2.41495250e-03,  1.76812094e-02,  1.61446761e-02],
        [ 1.11626936e-02,  1.80405262e-03, -3.70471133e-03, ...,
          5.23177721e-03, -2.92758900e-03,  1.04449764e-02],
        ...,
        [ 3.15350331e-02, -2.53555700e-02, -7.00413948e-03, ...,
         -1.19793406e-02, -4.30783257e-03,  7.22094486e-03],
        [ 1.97059624e-02, -3.16088088e-02, -2.43522599e-02, ...,
         -2.76908409e-02,  8.38099048e-03, -1.66237336e-02],
        [-2.70077102e-02, -1.11523010e-02, -1.38169136e-02, ...,
          7.15513655e-04, -4.87749884e-03,  2.62997858e-02]],

...,

[[ 5.38729597e-03,  3.36820409e-02,  2.64275914e-05, ...,
         -1.40761500e-02,  3.26351263e-02, -8.07712507e-03],
        [-5.79545237e-02,  1.27130207e-02, -4.27546017e-02, ...,
         -5.60793430e-02,  2.92317923e-02, -1.18598361e-02],
        [ 1.74755994e-02,  3.69822644e-02, -1.85797047e-02, ...,
          3.78087088e-02,  7.94491358e-03, -2.68882941e-02],
        ...,
        [ 1.57504007e-02, -2.90263388e-02, -2.07669195e-02, ...,
          2.18015630e-02,  5.82551258e-03,  2.76228762e-03],
        [ 9.64327808e-03, -6.16819635e-02,  2.22731056e-03, ...,
          4.87597985e-03,  1.76527463e-02, -7.40470458e-03],
        [ 4.91075814e-02, -2.84049120e-02, -1.76996272e-02, ...,
          1.68349165e-02,  1.24201104e-02,  5.73079512e-02]],

[[ 1.29527142e-02, -1.63868088e-02,  2.62253303e-02, ...,
          9.81368404e-03, -1.32641261e-02, -1.71046071e-02],
        [ 2.17389148e-02,  1.15475925e-02, -1.99689306e-02, ...,
          3.15210409e-02, -2.54344400e-02, -1.25245098e-02],
        [-8.03261704e-04, -7.89023377e-03, -5.84866107e-03, ...,
         -1.29525438e-02, -4.92958911e-03, -2.26518847e-02],
        ...,
        [-1.92850381e-02, -1.52473515e-02, -1.67591721e-02, ...,
          3.85184819e-03, -1.04960520e-03, -1.19505906e-02],
        [-1.25200218e-02, -1.56924445e-02, -4.84889792e-03, ...,
          8.76505766e-03,  3.59887863e-03,  2.09062565e-02],
        [ 8.13277438e-03, -9.65633523e-03, -4.39512804e-02, ...,
          3.05530336e-03,  3.83036304e-03, -9.42472275e-03]],

[[ 6.06435612e-02,  2.95455698e-02,  1.51243405e-02, ...,
          3.53831775e-03,  9.64399520e-03, -1.09638572e-02],
        [-4.68350127e-02,  3.44929323e-02,  2.58492399e-03, ...,
          8.91290884e-03, -2.92156115e-02,  3.20116431e-02],
        [ 3.71786840e-02, -3.70884401e-04,  1.91319417e-02, ...,
          1.18764145e-02,  5.01254341e-03,  5.30658336e-03],
        ...,
        [-3.02066002e-03, -1.16653452e-02, -3.25030982e-02, ...,
         -9.35567636e-03,  1.97983272e-02,  2.31254101e-02],
        [-3.90016064e-02, -2.88063362e-02, -9.27522033e-03, ...,
         -1.25017455e-02, -1.60281304e-02, -1.13976942e-02],
        [ 3.40925641e-02, -1.13470471e-02, -3.17137106e-03, ...,
         -5.07070217e-03,  1.89429813e-03,  3.31671350e-02]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.69457185, 0.849925  , 0.80576855, ..., 0.6025801 ,
         0.73450977, 0.6980251 ],
        [0.93835205, 0.7691214 , 0.92195517, ..., 0.89022326,
         1.0819645 , 0.791517  ],
        [0.5647757 , 0.7690196 , 0.82105225, ..., 0.7471048 ,
         0.75182617, 0.70143354],
        ...,
        [0.8081902 , 0.757499  , 0.7448052 , ..., 0.7828297 ,
         0.63044   , 0.6621047 ],
        [0.82019335, 0.94911325, 0.8245455 , ..., 0.799384  ,
         1.1135528 , 0.8962364 ],
        [0.8031752 , 0.8306218 , 0.76861745, ..., 0.8110981 ,
         0.7602989 , 0.8642122 ]]], dtype=float32)
                },
                'value': {
                    'b': array([[ 2.3536319e-03, -5.8421714e-04,  2.6541166e-03, ...,
        -1.1526708e-03,  1.2811483e-03, -6.3673397e-05],
       [-3.6600938e-03, -3.3384843e-03, -1.1473114e-03, ...,
        -1.4382317e-03, -3.8280077e-03,  5.3748143e-05],
       [-6.2878610e-04, -2.3630157e-03, -1.6891264e-03, ...,
        -4.5074243e-03,  2.0703450e-03,  1.6706111e-03],
       ...,
       [ 1.0337565e-03, -6.6874768e-03,  7.8707105e-03, ...,
         3.8281935e-03,  2.7246573e-03, -5.4889736e-03],
       [ 4.5474949e-03,  1.8496951e-03, -4.7515641e-04, ...,
         1.6249692e-03, -2.1909401e-03, -2.1236469e-03],
       [ 6.4964488e-04,  1.0762867e-03, -3.3429137e-03, ...,
         2.1054570e-03,  3.9049805e-04, -2.0609915e-03]], dtype=float32),
                    'w': array([[[-2.76697311e-03,  9.07628099e-04,  2.88689364e-04, ...,
          4.36796620e-03,  2.63751904e-03, -1.84235796e-02],
        [ 8.94250628e-03, -3.43922116e-02,  1.04978001e-02, ...,
          7.22295837e-03, -8.83479882e-03,  9.46309417e-03],
        [ 9.88198258e-03, -3.21986005e-02, -1.89686548e-02, ...,
         -1.78769864e-02, -1.25460494e-02,  1.63281579e-02],
        ...,
        [-1.66412315e-03, -8.80832225e-03,  1.40606370e-02, ...,
          3.45968977e-02,  4.83957585e-03,  1.87827908e-02],
        [ 4.24764538e-03, -2.04535387e-02,  2.49281321e-02, ...,
         -1.04700867e-02,  2.57115997e-02, -8.19846988e-03],
        [-1.17270052e-02, -9.50942002e-03, -2.19905041e-02, ...,
          2.44821776e-02, -7.87706580e-03, -1.02372207e-02]],

[[-1.96535178e-02, -1.92057174e-02,  6.18294254e-03, ...,
          3.90880788e-03, -8.71747825e-03, -1.33498181e-02],
        [-1.67660485e-03, -8.54353886e-03, -5.46948193e-03, ...,
          2.04944760e-02, -4.66637267e-03, -6.69365562e-03],
        [-7.27515062e-03, -1.31334073e-03,  6.16474869e-03, ...,
          3.08417226e-03, -1.50980251e-02, -9.36115393e-04],
        ...,
        [-1.33740706e-02,  9.02792718e-03,  4.10421472e-03, ...,
          1.65003203e-02,  8.73883441e-03, -7.25079468e-03],
        [-1.97785869e-02, -3.43597196e-02,  1.41484821e-02, ...,
         -3.88899702e-03, -4.25165817e-02, -2.82797758e-02],
        [-1.00196609e-02,  2.23649219e-02,  1.06529118e-02, ...,
         -1.05638793e-02, -6.38976507e-03,  1.07330894e-02]],

[[ 2.88438564e-03, -1.01825697e-02,  1.46885449e-02, ...,
          1.26958545e-02, -1.01610366e-02, -8.66008922e-03],
        [ 1.21528190e-02,  7.32445344e-03,  3.56625207e-03, ...,
          1.30250445e-02, -1.36456601e-02, -1.18230805e-02],
        [-4.03345982e-03,  2.06898927e-04,  7.74102844e-03, ...,
          5.27716707e-03,  2.41349917e-02, -2.12165783e-03],
        ...,
        [ 1.11693926e-02, -4.90604783e-04,  6.39192876e-04, ...,
          6.38136349e-04, -1.40210036e-02, -9.94266942e-03],
        [ 1.29607518e-03,  2.81136278e-02, -1.13350172e-02, ...,
         -6.19309908e-03, -2.01076665e-03, -6.14847289e-03],
        [ 1.22312969e-03, -1.08492104e-02,  2.09707096e-02, ...,
          3.05288727e-03, -9.85009782e-03, -8.65761563e-03]],

...,

[[-1.42929936e-02, -1.99877121e-03,  2.50426829e-02, ...,
          1.90337526e-03, -1.12309111e-02, -8.00351053e-03],
        [-2.69606849e-03,  1.09451320e-02, -2.93860361e-02, ...,
         -1.09698987e-02,  9.05686617e-03, -2.36088466e-02],
        [-4.60462086e-03, -2.74263113e-03, -6.74776558e-04, ...,
          8.27623997e-03, -1.68107133e-02,  2.63968459e-03],
        ...,
        [-6.02101954e-03, -2.39710063e-02,  3.02442629e-03, ...,
          6.68293610e-03,  6.07164577e-03,  1.18561927e-02],
        [ 1.62423670e-03,  1.35164093e-02,  1.07928151e-02, ...,
          8.94584227e-03,  9.46178474e-03,  9.24989767e-03],
        [-9.82824806e-03, -5.42457867e-03,  5.39344270e-03, ...,
          1.33040752e-02, -1.55275101e-02,  1.90090463e-02]],

[[-4.13861731e-03,  1.61855307e-03, -2.17836211e-03, ...,
          2.80943932e-03, -5.24073699e-03,  3.66205699e-03],
        [ 2.67545357e-02,  1.17547950e-03, -7.04694632e-03, ...,
         -1.49754016e-02,  3.96313611e-03,  1.77461293e-03],
        [-6.64195605e-03,  3.96287860e-03, -1.45245530e-03, ...,
         -7.52582448e-03, -1.52688392e-03,  4.50920127e-03],
        ...,
        [-1.31012069e-03, -3.52656818e-04,  1.31454132e-03, ...,
         -6.19313773e-03, -1.56298727e-02, -1.35568185e-02],
        [-2.56815623e-03,  5.28231356e-03,  5.83310518e-03, ...,
         -1.22300908e-02,  2.77721114e-03,  7.60130631e-03],
        [-7.02921720e-03,  1.38298692e-02,  7.83662498e-03, ...,
          1.07167484e-02, -1.19849928e-02,  1.03575205e-02]],

[[ 2.47591967e-03, -2.99220229e-03,  4.34108218e-03, ...,
          6.08616602e-03,  7.70103559e-03,  4.92395193e-04],
        [-2.45214876e-04, -4.27352125e-03,  2.37497184e-02, ...,
          1.16175665e-02, -8.89002532e-03,  1.17512504e-02],
        [ 5.56151150e-03, -2.24482492e-02,  4.73437086e-03, ...,
          6.34752354e-03,  1.10943178e-02,  3.15959100e-03],
        ...,
        [ 1.53137231e-02, -2.43450031e-02,  2.87173828e-03, ...,
          3.01783346e-02,  1.53354611e-02,  6.15234673e-03],
        [-1.02614937e-02,  1.27734924e-02,  9.20217112e-03, ...,
         -1.02896625e-02,  1.91032458e-02,  2.17168345e-05],
        [-1.01738321e-02, -6.64736098e-03, -4.09644879e-02, ...,
          9.67943482e-03,  1.93070676e-02,  9.33439843e-03]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.36925334, 0.3515637 , 0.48469514, ..., 0.49889025,
         0.47127423, 0.4275097 ],
        [0.4616928 , 0.5203079 , 0.52224624, ..., 0.5284845 ,
         0.4929024 , 0.51608026],
        [0.42673588, 0.49105448, 0.49275902, ..., 0.4043688 ,
         0.50152534, 0.46596774],
        ...,
        [0.43131086, 0.41935384, 0.34408888, ..., 0.44367346,
         0.46299422, 0.48696527],
        [0.5134071 , 0.5025146 , 0.49936026, ..., 0.35289165,
         0.48807734, 0.4320891 ],
        [0.44177976, 0.4690712 , 0.4662832 , ..., 0.47841358,
         0.49142972, 0.47127607]]], dtype=float32)
                }
            }
        },
        'x_layers_17': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': array([ 0.00171706, -0.00131204, -0.0021915 , ..., -0.00766173,
        0.00270988, -0.00303804], dtype=float32)
                    },
                    'linear': {
                        'w': array([[ 0.02472598,  0.00262219,  0.05628514, ..., -0.04744835,
        -0.05583418,  0.000448  ],
       [ 0.0435979 , -0.03077106, -0.01843363, ..., -0.03928912,
         0.00057252, -0.00394176],
       [ 0.02986974, -0.03444536, -0.03493382, ...,  0.0275128 ,
         0.00404201, -0.03031758],
       ...,
       [-0.01906034,  0.00683718, -0.02937934, ...,  0.02432913,
        -0.02658359, -0.02528914],
       [ 0.02618262, -0.03669364,  0.0492231 , ..., -0.0325532 ,
        -0.00535654,  0.03550595],
       [ 0.01585514,  0.01979536,  0.02232511, ..., -0.00242681,
        -0.02909306, -0.05983097]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[0.99977  , 0.9552387, 1.1004109, ..., 1.0892595, 0.9593381,
        0.9522808]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': array([-0.00295017,  0.00484709,  0.00511358, ..., -0.00108915,
       -0.00493497, -0.00364521], dtype=float32)
                    },
                    'linear': {
                        'w': array([[ 0.00931492, -0.01786293,  0.02748417, ..., -0.0600135 ,
        -0.01249322,  0.02373556],
       [ 0.03530489,  0.00356561,  0.04185544, ..., -0.02650248,
        -0.0146055 , -0.05268423],
       [ 0.03967872, -0.0358183 ,  0.0471034 , ..., -0.03817839,
         0.02537379, -0.06090042],
       ...,
       [ 0.0485337 ,  0.00673793, -0.02475552, ..., -0.01539302,
         0.01099681,  0.01217936],
       [ 0.03491158,  0.03560209,  0.0085284 , ..., -0.01987701,
         0.02958684,  0.03682547],
       [-0.04517267, -0.02559242,  0.04155496, ..., -0.03713605,
        -0.01925383,  0.02169372]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.3913367, 1.3241429, 1.2769357, ..., 1.3841518, 1.4442827,
        1.4156634]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': array([ 0.0058205 , -0.00151459,  0.0105388 , ..., -0.00131459,
       -0.02444017,  0.00267917], dtype=float32),
                    'scale': array([0.01300202, 0.02647691, 0.02123367, ..., 0.01254215, 0.06005605,
       0.01016154], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': array([1.1104951, 1.1179245, 1.0624976, ..., 1.1053315, 1.0585247,
       1.0821815], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': array([[ 0.00256548, -0.00106355,  0.00024083, ..., -0.00201515,
         0.00219756, -0.00327231],
       [ 0.00121145, -0.00157773,  0.00120874, ..., -0.00152556,
         0.00318669, -0.00010758],
       [-0.00213632,  0.00069429,  0.00132417, ...,  0.00021959,
        -0.00080004, -0.0014997 ],
       ...,
       [-0.0032619 , -0.00237538,  0.00118759, ...,  0.00123081,
         0.00139293, -0.00295219],
       [ 0.00184485,  0.000301  ,  0.00165043, ..., -0.00173555,
         0.00077969,  0.00220971],
       [ 0.000826  ,  0.00090565, -0.00069976, ..., -0.00110533,
        -0.00024552, -0.00129298]], dtype=float32),
                    'w': array([[[ 0.00062362, -0.01420906,  0.00070949, ...,  0.01102791,
          0.01875118, -0.00241041],
        [-0.01302008, -0.01643992, -0.00820376, ..., -0.01269134,
         -0.01051504, -0.00433114],
        [ 0.01605858,  0.03422265, -0.01969025, ..., -0.02132047,
         -0.01710039, -0.00646175],
        ...,
        [-0.01284659, -0.027529  , -0.009594  , ..., -0.02514833,
          0.00789621, -0.02574673],
        [-0.01678951, -0.00012382, -0.00577383, ..., -0.00447377,
          0.01541107,  0.03425353],
        [-0.00548437, -0.00744928, -0.0070674 , ...,  0.02867692,
          0.0161877 ,  0.00638222]],

[[-0.01049402,  0.0156538 , -0.00325412, ..., -0.01518418,
         -0.01721447, -0.02334522],
        [ 0.01972598, -0.02769345,  0.01937203, ...,  0.01346674,
          0.00637783,  0.02078615],
        [-0.00083938,  0.01387903, -0.00579068, ...,  0.00690409,
         -0.02381891, -0.01517515],
        ...,
        [ 0.0030487 ,  0.00637952, -0.01281735, ...,  0.00226798,
         -0.00167745, -0.00779316],
        [ 0.01851878,  0.02485802, -0.0108985 , ..., -0.00518591,
          0.00952113, -0.00941548],
        [-0.00038227,  0.00331443,  0.01079975, ...,  0.00527024,
          0.00147566, -0.00832753]],

[[ 0.01593056,  0.00281412, -0.03783843, ..., -0.02565233,
          0.03177755, -0.01711301],
        [ 0.00578596,  0.04798891, -0.0207148 , ..., -0.00409601,
         -0.02728604,  0.03273825],
        [ 0.00588926,  0.00621151,  0.00162665, ...,  0.00728724,
         -0.01068193,  0.00820528],
        ...,
        [-0.0068747 ,  0.00570187, -0.0002149 , ..., -0.00771358,
          0.01695905,  0.00598655],
        [-0.01190429, -0.00825517, -0.02323935, ...,  0.01609944,
         -0.00844449,  0.01326131],
        [-0.01609641,  0.03854297,  0.0199534 , ..., -0.02685795,
         -0.02703301,  0.00840214]],

...,

[[ 0.00150173, -0.00993695, -0.03154258, ...,  0.00618782,
         -0.00278643, -0.00608832],
        [ 0.01758712,  0.00107316,  0.03609834, ...,  0.01897288,
         -0.01170872,  0.00389629],
        [ 0.01152163,  0.01028328,  0.00493881, ..., -0.004137  ,
         -0.0037651 , -0.02994725],
        ...,
        [ 0.00724428,  0.00378446, -0.00150858, ...,  0.01981086,
          0.01538553, -0.01338076],
        [ 0.00512812,  0.02182308,  0.02148484, ...,  0.01195069,
          0.00624322, -0.0085104 ],
        [-0.02870601,  0.00980159,  0.01508211, ...,  0.00408035,
         -0.01716553,  0.01633615]],

[[ 0.01714331,  0.02185955, -0.02319431, ..., -0.01634747,
         -0.00199004,  0.00107747],
        [ 0.00950827, -0.00539518, -0.00959258, ..., -0.01569375,
         -0.00954814, -0.01278928],
        [ 0.00041261,  0.00235445,  0.00362286, ...,  0.01036649,
          0.00770445,  0.00961575],
        ...,
        [-0.0066833 , -0.0004319 ,  0.01164088, ..., -0.0183213 ,
         -0.00688071,  0.0131796 ],
        [ 0.00323726, -0.00399138,  0.0101805 , ..., -0.00067636,
         -0.00884127,  0.01011052],
        [ 0.01870758,  0.00702295, -0.01884997, ..., -0.00614319,
         -0.01268861, -0.00230975]],

[[ 0.00251885,  0.0151092 , -0.00846976, ...,  0.02188397,
         -0.00930506,  0.01001358],
        [-0.0057018 , -0.01241554,  0.00730267, ...,  0.02563364,
          0.01692494,  0.00170398],
        [ 0.01367783,  0.00302956, -0.00021606, ..., -0.00508406,
         -0.01428949, -0.02000578],
        ...,
        [ 0.00197864, -0.0168051 , -0.00532901, ..., -0.00698534,
          0.00512153,  0.00303917],
        [-0.00151376, -0.00486665, -0.01476791, ..., -0.00450136,
          0.00636599, -0.01712616],
        [ 0.00871773, -0.00984234, -0.01420589, ..., -0.00379258,
          0.00297406, -0.01527084]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.5482985 , 0.5254131 , 0.57317084, ..., 0.5439772 ,
         0.5798373 , 0.5424537 ],
        [0.6962211 , 0.65189624, 0.53643084, ..., 0.585327  ,
         0.5911653 , 0.53630084],
        [0.48982716, 0.51645213, 0.5058164 , ..., 0.44775844,
         0.4472789 , 0.4628983 ],
        ...,
        [0.40842137, 0.51651293, 0.5592008 , ..., 0.48496637,
         0.46706876, 0.543186  ],
        [0.4809265 , 0.5128039 , 0.49324033, ..., 0.5848907 ,
         0.579181  , 0.55237156],
        [0.50562274, 0.6872567 , 0.49481732, ..., 0.5886361 ,
         0.43811685, 0.5983461 ]]], dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': array([0.18329173, 0.306529  , 0.33626303, 0.34032774, 0.31197897,
       0.26566285, 0.21109979, 0.2993935 , 0.3012725 , 0.13591361,
       0.25008982, 0.17057759, 0.33780378, 0.19163084, 0.16780691,
       0.3133621 , 0.29482302, 0.29536977, 0.27895793, 0.25672314,
       0.29341793, 0.32210267, 0.29777268, 0.18360291, 0.3179472 ,
       0.29114604, 0.3051625 , 0.25520694, 0.21351007, 0.30364794,
       0.27713385, 0.32244632, 0.27059892, 0.17526585, 0.2978192 ,
       0.26505676, 0.31563473, 0.22453122, 0.2727974 , 0.1579615 ,
       0.14100088, 0.31131014, 0.22335987, 0.28845438, 0.23881653,
       0.25362653, 0.3082384 , 0.2262864 , 0.2552006 , 0.24002545,
       0.18481648, 0.2526903 , 0.22395182, 0.1485628 , 0.33651558,
       0.24711472, 0.2915276 , 0.25388318, 0.27354527, 0.25630867,
       0.21607193, 0.30499017, 0.2570673 , 0.2858135 , 0.21992019,
       0.28136933, 0.16238771, 0.28450722, 0.22485426, 0.33766568,
       0.15231392, 0.25662956, 0.26807204, 0.29038495, 0.3233309 ,
       0.17098823, 0.33013552, 0.27603385, 0.27568522, 0.31669253],
      dtype=float32)
                },
                'post': {
                    'b': array([-0.00257743,  0.00719224,  0.01381382, ...,  0.00106568,
       -0.01411214, -0.0005999 ], dtype=float32),
                    'w': array([[[ 2.23880429e-02,  1.80971122e-03,  4.94497735e-03, ...,
          7.46185472e-03, -1.91377718e-02,  5.51110785e-03],
        [-9.65329073e-03, -2.50214022e-02,  6.51374366e-03, ...,
          7.30558007e-04, -1.10678503e-03,  1.15562894e-03],
        [-1.30426968e-02, -1.25828469e-02, -8.35072715e-03, ...,
         -5.25994552e-03, -2.06049960e-02,  1.01154838e-02],
        ...,
        [-3.40172695e-03,  1.04015395e-02,  2.16603074e-02, ...,
          1.59501017e-03, -9.02491342e-03, -4.56762221e-03],
        [ 1.19904121e-02, -1.82692781e-02, -1.07434206e-03, ...,
         -7.59373792e-03,  4.29450534e-03, -4.55015711e-03],
        [ 1.28748650e-02, -6.32785633e-03,  1.58232097e-02, ...,
         -1.22512644e-02,  2.13666223e-02,  3.45551148e-02]],

[[-1.05541665e-02, -1.11681661e-02, -2.31585442e-03, ...,
         -7.87847862e-03,  5.69366151e-03, -5.99862961e-03],
        [-9.61955078e-03, -4.94371960e-03,  1.41007034e-02, ...,
          3.07045644e-03, -2.70487294e-02,  9.97978449e-03],
        [ 3.76466708e-03,  8.85370187e-03, -1.38930622e-02, ...,
          1.21522751e-02, -6.59066299e-03,  2.39906553e-02],
        ...,
        [ 5.87391248e-03,  1.07618440e-02,  1.12370299e-02, ...,
         -7.07917474e-03,  4.27670777e-03, -2.10730912e-04],
        [ 9.23075771e-04, -7.51872035e-03,  2.25937064e-03, ...,
          2.92556686e-03,  9.63122770e-03, -9.21652419e-04],
        [-7.78586185e-03,  8.27838574e-03,  6.34420943e-03, ...,
          6.03399938e-03,  7.64258625e-03,  4.97977016e-04]],

[[ 1.22948000e-02,  7.53727218e-04, -7.88786449e-03, ...,
          1.77575427e-03, -5.48499357e-03,  1.22255515e-02],
        [ 8.92442837e-03, -1.33555299e-02, -6.57343830e-04, ...,
          1.46213528e-02,  1.23695740e-02,  4.33483440e-03],
        [-8.68047308e-03, -1.26249064e-02, -2.08581774e-03, ...,
         -2.64701601e-02, -8.13051499e-03, -1.05354162e-02],
        ...,
        [ 2.13066000e-03,  6.05899654e-03,  1.01602934e-02, ...,
         -7.75652612e-03,  8.76510516e-03,  1.08143464e-02],
        [-1.32632963e-02,  2.31360341e-03, -1.21273873e-02, ...,
          6.44184602e-03, -7.98467919e-03, -6.96573220e-03],
        [ 3.10448464e-03,  6.87763840e-03, -1.00372601e-02, ...,
         -2.23641749e-02,  1.76847179e-03, -1.28047988e-02]],

...,

[[ 1.04766572e-02,  5.13660198e-04, -4.88733081e-03, ...,
          2.68076006e-02,  6.38087140e-03, -1.33746315e-03],
        [ 2.68360842e-02,  3.05081047e-02,  2.75111441e-02, ...,
         -3.97647657e-02,  2.93309093e-02, -1.10622430e-02],
        [-1.14255643e-03,  1.96056832e-02,  2.31193961e-03, ...,
         -7.19036395e-03, -7.56851677e-03, -3.88612552e-03],
        ...,
        [-7.26628816e-03,  2.16381960e-02,  6.46241067e-04, ...,
         -2.86439201e-03,  4.67671128e-03, -2.15039682e-02],
        [ 3.94482538e-03,  8.57019611e-03,  1.69229731e-02, ...,
         -7.88632873e-03, -8.91969074e-03,  2.49818321e-02],
        [-4.37163748e-04, -2.38167704e-03,  6.50402997e-03, ...,
          5.00184437e-03,  7.39515340e-03,  4.14480455e-03]],

[[-1.52585339e-02,  1.64111072e-04,  1.17837843e-02, ...,
          8.56793858e-03,  9.24884155e-03,  2.94837635e-03],
        [ 1.87203251e-02,  2.36569042e-03, -2.45390031e-02, ...,
         -1.55704739e-02, -4.62393090e-03, -3.53215705e-03],
        [-7.32748769e-04,  2.07418688e-02,  6.12184172e-03, ...,
          1.99002828e-02,  1.51653364e-02, -4.16355114e-03],
        ...,
        [ 5.55806607e-03, -8.35503265e-03, -1.65117551e-02, ...,
          1.07367812e-02,  5.08033438e-03,  3.46588809e-03],
        [ 4.88130236e-03, -5.42979641e-03, -1.38777941e-02, ...,
         -1.93922725e-02, -5.02611836e-03,  3.43181775e-03],
        [-1.13557000e-02,  1.11429971e-02, -4.90684761e-03, ...,
         -3.52203194e-03,  9.77618061e-03, -1.21046836e-02]],

[[-5.59269032e-03,  9.68667027e-03,  1.00202777e-03, ...,
         -3.58658191e-03,  1.04861651e-02,  4.70099360e-04],
        [ 9.24368761e-03, -7.87344668e-03,  8.12148664e-06, ...,
          9.53086000e-03, -3.83773400e-03,  2.40035001e-02],
        [-1.04424749e-02,  2.85305525e-03, -1.21543240e-02, ...,
          2.15355158e-02, -6.34655217e-03,  1.02934204e-02],
        ...,
        [ 3.62815685e-04,  3.94196855e-03,  8.03928822e-03, ...,
         -2.90077459e-03, -5.93607372e-04,  3.15086567e-03],
        [-1.50619149e-02, -2.51054447e-02, -2.12774873e-02, ...,
         -3.54218693e-03,  2.81670527e-03, -1.14895822e-02],
        [ 1.07423374e-02, -7.61732575e-04,  3.77989337e-02, ...,
          9.42581892e-03,  1.79382302e-02, -4.48567374e-03]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.41280448, 0.3319148 , 0.36389962, ..., 0.5233173 ,
         0.5465536 , 0.4390002 ],
        [0.3794163 , 0.4581972 , 0.53180695, ..., 0.4617975 ,
         0.64440525, 0.45324442],
        [0.34979746, 0.47479686, 0.35922474, ..., 0.44793785,
         0.34815326, 0.4603146 ],
        ...,
        [0.47263524, 0.47230673, 0.411919  , ..., 0.3656408 ,
         0.33189306, 0.34749904],
        [0.4995878 , 0.543227  , 0.38369125, ..., 0.45565164,
         0.34350705, 0.53263223],
        [0.5062739 , 0.4824583 , 0.40826628, ..., 0.5135839 ,
         0.4950239 , 0.4851064 ]]], dtype=float32)
                },
                'query': {
                    'b': array([[ 0.01000108,  0.01639372,  0.00138619, ...,  0.00279542,
         0.01172931, -0.01006246],
       [-0.00861747, -0.00524879,  0.01512013, ..., -0.01792894,
         0.0165606 ,  0.01120927],
       [ 0.01261996,  0.0112497 , -0.00451962, ..., -0.01455342,
        -0.01891823, -0.01427975],
       ...,
       [-0.01672397, -0.00771439,  0.00280647, ..., -0.0115596 ,
         0.00929647, -0.00382957],
       [ 0.0151234 ,  0.01519945,  0.03095485, ..., -0.01198111,
         0.00296444,  0.01389938],
       [-0.01285456, -0.00179988,  0.00022462, ..., -0.00894534,
        -0.01765853, -0.00730262]], dtype=float32),
                    'w': array([[[-0.02677594,  0.00599007, -0.00895208, ..., -0.00217276,
          0.0255127 , -0.00099586],
        [ 0.00824958,  0.00178674, -0.01930125, ..., -0.04208021,
         -0.00241481, -0.04280047],
        [ 0.03461367,  0.02654499, -0.01111733, ..., -0.00716423,
          0.02020551, -0.05437608],
        ...,
        [-0.02426664, -0.01644287,  0.00571987, ..., -0.04034445,
          0.0047169 , -0.01945657],
        [ 0.00492201, -0.03032579,  0.01060695, ..., -0.03597231,
          0.04792401, -0.00643607],
        [ 0.0501137 ,  0.0061601 , -0.03792244, ..., -0.03496204,
         -0.02160517,  0.0073292 ]],

[[ 0.00235153,  0.01533699, -0.01475207, ...,  0.02629482,
          0.02120748, -0.05778363],
        [-0.00327274, -0.03517571,  0.02899981, ...,  0.02759661,
          0.00555469,  0.02377838],
        [-0.01424948,  0.01438333, -0.04205852, ..., -0.01065959,
         -0.01528539,  0.01438235],
        ...,
        [-0.00972229, -0.03295676, -0.01328224, ...,  0.0007147 ,
          0.0013666 , -0.01047199],
        [-0.02229727,  0.03404536, -0.01399656, ...,  0.00739281,
          0.0164062 , -0.00740389],
        [-0.02559691, -0.01718573, -0.00239006, ...,  0.0031849 ,
         -0.0392669 ,  0.00416007]],

[[ 0.01108718,  0.0428006 , -0.0326011 , ..., -0.00583586,
          0.05997794, -0.00215854],
        [-0.03940746, -0.00041668, -0.02188808, ..., -0.03580334,
         -0.01030845,  0.00874109],
        [ 0.01737563,  0.02734391, -0.00097198, ...,  0.00236457,
          0.00346533, -0.01029301],
        ...,
        [-0.00208987,  0.01385075,  0.0216072 , ..., -0.00106191,
          0.01983339,  0.00400108],
        [-0.02079791, -0.01839657, -0.00752878, ..., -0.01003886,
         -0.04057409,  0.05555528],
        [-0.04144968,  0.00824266,  0.04001064, ..., -0.02460342,
         -0.0102106 , -0.00717507]],

...,

[[ 0.01884005,  0.03520181,  0.00118039, ..., -0.03736692,
          0.05458127,  0.04947542],
        [ 0.00188771,  0.01234467,  0.00955034, ...,  0.01729519,
         -0.00513513, -0.01496484],
        [ 0.00566824, -0.0101429 , -0.01463831, ...,  0.02097524,
          0.01507939,  0.04670054],
        ...,
        [ 0.00687755,  0.00456142, -0.00131266, ...,  0.00416755,
          0.00091567,  0.01513379],
        [ 0.00649664, -0.00748601, -0.00860857, ...,  0.02136184,
          0.02535005, -0.00031381],
        [-0.03581821,  0.00238554,  0.02519265, ..., -0.02932049,
         -0.02980674, -0.02002719]],

[[ 0.03207592,  0.01381679,  0.01308377, ...,  0.0031753 ,
          0.03006743,  0.01074742],
        [ 0.01111525,  0.00370368, -0.01082065, ...,  0.00090454,
          0.01673884, -0.00459667],
        [-0.00266687,  0.00602411, -0.00977136, ...,  0.03008852,
          0.02318002,  0.01576366],
        ...,
        [ 0.00350331,  0.0049806 ,  0.00743642, ...,  0.00894128,
         -0.02724188, -0.01732498],
        [ 0.02852351, -0.00478848, -0.02108903, ...,  0.02688607,
          0.01538384, -0.05201942],
        [ 0.04519854,  0.00514203,  0.0053154 , ...,  0.02465764,
          0.0175005 , -0.03822153]],

[[-0.0056584 , -0.03200392, -0.01644075, ..., -0.03680464,
         -0.00148697,  0.02020196],
        [ 0.00363839,  0.03369744, -0.02185607, ...,  0.02236145,
         -0.04222722,  0.00038236],
        [ 0.05848982, -0.01653695, -0.0145413 , ...,  0.0178215 ,
          0.01025046, -0.04149369],
        ...,
        [ 0.01977103, -0.00522108, -0.01307487, ...,  0.03509489,
         -0.00274744, -0.00983507],
        [-0.0104529 , -0.01347402, -0.0346458 , ..., -0.02915582,
          0.03944222,  0.00634644],
        [ 0.01344375,  0.00525091, -0.03533768, ...,  0.00530439,
         -0.01809248,  0.00764443]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.85786974, 0.85174143, 0.7332705 , ..., 0.6393211 ,
         0.9094431 , 0.813887  ],
        [0.6589663 , 0.7280409 , 0.8100041 , ..., 0.712833  ,
         0.70534873, 0.70867157],
        [0.5920055 , 0.6463384 , 0.68925416, ..., 0.58952117,
         0.72404957, 0.76216185],
        ...,
        [0.6576804 , 0.7099671 , 0.75340176, ..., 0.78092164,
         0.7273401 , 0.6942459 ],
        [0.6729461 , 0.6623584 , 0.8342663 , ..., 0.7712819 ,
         0.7314669 , 0.80964786],
        [0.78999907, 0.707104  , 0.8686639 , ..., 0.7699812 ,
         0.8380833 , 0.6055745 ]]], dtype=float32)
                },
                'value': {
                    'b': array([[ 1.6401194e-03,  5.3922208e-03, -5.0728004e-03, ...,
        -1.8111293e-03,  3.5093396e-04, -2.6095917e-03],
       [-1.7891753e-03, -7.4543618e-04, -1.1999641e-05, ...,
        -2.5042202e-03, -4.3558124e-03,  2.0704704e-04],
       [-1.1988505e-03, -1.7449020e-03,  5.3674653e-03, ...,
        -8.3908770e-04, -2.0188882e-03,  1.9707580e-03],
       ...,
       [-3.9382582e-03, -1.8472752e-03,  2.7255272e-03, ...,
         1.2182797e-03, -1.7120343e-05, -9.9033071e-04],
       [-2.3493757e-03, -6.2046667e-05,  2.9998010e-03, ...,
         2.6661516e-03, -3.9518168e-03, -1.3634117e-03],
       [ 2.6297674e-03,  2.2432012e-03,  9.0688950e-04, ...,
         2.9568389e-04, -4.6239821e-03, -2.7873907e-03]], dtype=float32),
                    'w': array([[[-0.00406329,  0.00125176, -0.00566959, ...,  0.02132011,
          0.00988541,  0.03411871],
        [-0.00466405,  0.01822393,  0.03936379, ..., -0.01542882,
          0.02916765, -0.00973647],
        [ 0.00336628,  0.01916592,  0.01381161, ...,  0.0287559 ,
          0.00210257, -0.00523479],
        ...,
        [-0.00771339, -0.00339798, -0.01955114, ..., -0.00185548,
          0.00454612,  0.00495689],
        [-0.00247559,  0.0023475 , -0.00615943, ..., -0.01418786,
          0.00403497, -0.01032023],
        [ 0.01253546,  0.01948996,  0.00612835, ...,  0.03005783,
         -0.0113332 ,  0.00066234]],

[[ 0.01160568,  0.00716185,  0.00936279, ...,  0.03318275,
         -0.0199285 , -0.00058041],
        [-0.01194467, -0.01533656, -0.00240232, ...,  0.00208076,
          0.01198673,  0.01881663],
        [-0.00071184,  0.00089308,  0.00799544, ...,  0.00168967,
          0.00223725, -0.01011235],
        ...,
        [ 0.0026001 , -0.00106251, -0.01970693, ..., -0.01094563,
         -0.01064124,  0.00068932],
        [-0.00765219,  0.01941295, -0.0049519 , ..., -0.01431958,
          0.0086679 , -0.00963882],
        [-0.01503419,  0.00209293, -0.0059628 , ..., -0.02016142,
         -0.02047891, -0.02072661]],

[[-0.00688163,  0.00503705,  0.00033843, ..., -0.02899964,
         -0.01288056, -0.00256081],
        [ 0.00336693,  0.0008492 ,  0.00208588, ...,  0.01013987,
         -0.01709507,  0.00924823],
        [ 0.00954575,  0.00369925,  0.00047883, ...,  0.00099918,
         -0.00079574, -0.00532331],
        ...,
        [-0.00523515, -0.02289669, -0.0149868 , ..., -0.00367369,
         -0.0022146 ,  0.0058173 ],
        [ 0.00210556,  0.00856084, -0.00385807, ..., -0.00279999,
          0.01040069, -0.00465724],
        [-0.00629756, -0.00418553, -0.00578899, ...,  0.00350722,
          0.00580828, -0.00013111]],

...,

[[-0.01655662,  0.00776426, -0.00594314, ..., -0.00156803,
          0.01890394, -0.00878979],
        [ 0.00585801, -0.02201295, -0.02875423, ..., -0.00845404,
         -0.02983253, -0.00885262],
        [ 0.00656099, -0.03694526,  0.00652788, ..., -0.00845556,
          0.00637956,  0.0099696 ],
        ...,
        [ 0.01514464, -0.00970265,  0.00530792, ..., -0.00570929,
          0.00818836, -0.00398522],
        [-0.00424537,  0.01111172, -0.00043121, ...,  0.0009518 ,
         -0.0114273 , -0.00251304],
        [-0.00099156,  0.01033743,  0.03074737, ...,  0.00525204,
          0.01159419, -0.00989561]],

[[ 0.00503532, -0.01248188,  0.00907086, ..., -0.00565056,
         -0.0002736 , -0.00475058],
        [-0.01139254, -0.02553086,  0.02108413, ...,  0.00414069,
          0.01168257, -0.00615824],
        [-0.00264056, -0.00835863, -0.00458868, ...,  0.00861508,
          0.00371939, -0.01088262],
        ...,
        [ 0.00403074, -0.00644513,  0.00213644, ..., -0.01067729,
         -0.00109866,  0.00979602],
        [-0.00958341, -0.01889652,  0.01720979, ...,  0.002966  ,
          0.01135705, -0.00930955],
        [-0.01068717, -0.00643605, -0.00832345, ..., -0.01019123,
          0.01236548, -0.01610381]],

[[-0.0159568 ,  0.00502084, -0.02019401, ...,  0.00596474,
          0.0236849 , -0.02418589],
        [ 0.02489095,  0.02883545, -0.00887448, ...,  0.00603098,
          0.0235995 , -0.01852634],
        [ 0.01750358,  0.00283267,  0.01605931, ...,  0.00164372,
          0.01845782, -0.00467066],
        ...,
        [ 0.01845044,  0.00277444, -0.00401578, ...,  0.01248607,
         -0.00714417, -0.01576691],
        [ 0.00713076,  0.02249625,  0.00833604, ...,  0.02588477,
         -0.00887445,  0.0239369 ],
        [ 0.0180273 ,  0.00425099, -0.00061115, ...,  0.01271264,
          0.0059908 , -0.00427213]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.42095944, 0.3483154 , 0.3689971 , ..., 0.4966507 ,
         0.54278594, 0.42725933],
        [0.37380052, 0.43494147, 0.5292852 , ..., 0.48110864,
         0.578429  , 0.4360593 ],
        [0.349415  , 0.45360166, 0.41077656, ..., 0.4690664 ,
         0.34185335, 0.41670692],
        ...,
        [0.48668742, 0.4711633 , 0.4351654 , ..., 0.3586824 ,
         0.3355607 , 0.35116765],
        [0.4910687 , 0.50001085, 0.38101885, ..., 0.4223548 ,
         0.34675717, 0.50167793],
        [0.4323144 , 0.47634688, 0.41576982, ..., 0.5221792 ,
         0.50473785, 0.47247168]]], dtype=float32)
                }
            }
        },
        'x_layers_18': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': array([ 0.00051136, -0.01148895,  0.00169411, ..., -0.00297784,
       -0.00223244, -0.0024133 ], dtype=float32)
                    },
                    'linear': {
                        'w': array([[ 0.00378705, -0.04327458,  0.07254019, ..., -0.04718518,
         0.03434655,  0.0322656 ],
       [ 0.02538404,  0.02334474, -0.03720424, ...,  0.01903304,
        -0.01485185,  0.04608155],
       [ 0.02892762, -0.04424008, -0.02908011, ..., -0.07568386,
        -0.02110019, -0.01552906],
       ...,
       [ 0.03719405, -0.02541165,  0.05467693, ..., -0.08789484,
        -0.02933055, -0.00520746],
       [-0.03056414, -0.00606433,  0.08469906, ...,  0.09002001,
         0.02983194,  0.02113312],
       [ 0.0071219 , -0.00418697,  0.07024559, ..., -0.06116322,
        -0.05130556, -0.04037147]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[0.99485636, 1.3522916 , 1.5606422 , ..., 1.354201  , 1.1416366 ,
        1.0600771 ]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': array([-0.00182306,  0.00342986,  0.00286894, ..., -0.005461  ,
       -0.00311243,  0.00018258], dtype=float32)
                    },
                    'linear': {
                        'w': array([[ 0.01468336,  0.02432303, -0.00566332, ..., -0.03545314,
         0.00487696,  0.03696332],
       [-0.00037203,  0.0256752 ,  0.02414542, ...,  0.06217912,
         0.00418166, -0.07536559],
       [ 0.00479466, -0.02747266, -0.04529685, ..., -0.03484176,
         0.05739212,  0.09449966],
       ...,
       [-0.06952304,  0.05575815,  0.00547075, ..., -0.01663679,
         0.10182662, -0.08988286],
       [ 0.03011166,  0.02201597, -0.0587035 , ..., -0.02506955,
         0.03673257,  0.02384413],
       [-0.03572247,  0.02936964, -0.0085162 , ...,  0.03586204,
        -0.00344068,  0.0381567 ]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.4908044, 1.5091405, 1.3977795, ..., 1.5834904, 1.3851438,
        1.4310642]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': array([-1.6253753e-04,  2.0698961e-05,  4.9653015e-04, ...,
        1.8806861e-03, -7.9710046e-03, -6.5031382e-03], dtype=float32),
                    'scale': array([0.03690767, 0.01909823, 0.0559881 , ..., 0.04496968, 0.05597709,
       0.0311148 ], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': array([1.1104548, 1.1232488, 1.1060338, ..., 1.0945585, 1.1172719,
       1.0616626], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': array([[ 4.62999305e-04, -9.51305556e-04, -7.77834328e-04, ...,
        -1.01630467e-04, -1.42782566e-03, -1.38255733e-03],
       [-3.89249367e-03,  1.00802258e-03, -3.89283267e-03, ...,
        -1.02095969e-03,  4.24970029e-04,  6.21549669e-04],
       [ 1.71094539e-03, -2.22824907e-04, -2.29796919e-04, ...,
         6.06717891e-04, -1.62431935e-03,  8.83733795e-04],
       ...,
       [ 1.87748508e-03,  4.73401946e-04, -5.41095797e-04, ...,
        -1.95898069e-03, -4.02874604e-04, -1.82685454e-03],
       [ 2.49577849e-03, -6.80000638e-04,  4.40471398e-04, ...,
         5.82880108e-04,  3.10799922e-03,  4.94156266e-04],
       [-1.02130929e-03,  5.47624040e-05, -5.23315066e-05, ...,
        -1.04511355e-03, -3.17567843e-04,  1.13948445e-04]], dtype=float32),
                    'w': array([[[-1.25487559e-02,  5.27682714e-03, -8.67328432e-04, ...,
         -1.05484826e-02,  6.49900082e-03, -1.18660368e-02],
        [-6.30257931e-03,  5.88576403e-03, -2.17002351e-03, ...,
          2.14710347e-02,  1.67809043e-03, -1.66181996e-02],
        [ 4.71557165e-03, -3.11150122e-03, -1.95540395e-02, ...,
          4.38414188e-03, -8.43202753e-04,  3.24525163e-02],
        ...,
        [-7.45568192e-04,  1.49174854e-02, -1.20049333e-02, ...,
          2.66597308e-02, -5.00853918e-03,  1.92217473e-02],
        [-4.22099652e-03, -1.88382305e-02,  2.46013161e-02, ...,
          1.17878467e-02, -5.46755921e-03,  1.35662928e-02],
        [-1.58921070e-02,  1.14033790e-02,  9.26128123e-03, ...,
         -1.08587695e-02,  1.97254512e-02, -3.81363183e-02]],

[[ 2.60815248e-02,  1.59908962e-02,  3.13113146e-02, ...,
         -7.63030350e-03,  2.86262929e-02,  5.76356938e-03],
        [ 1.09933987e-02, -3.30818109e-02, -2.09616777e-03, ...,
         -1.46984132e-02, -7.30580324e-03,  8.45863018e-03],
        [-7.00786430e-03, -2.33726092e-02,  3.69497784e-03, ...,
         -1.38930054e-02,  5.81548689e-03,  1.02452645e-02],
        ...,
        [-2.51502618e-02, -2.12524254e-02,  2.37490088e-02, ...,
          5.43616377e-02, -7.78818084e-03,  9.22023319e-03],
        [-4.27098852e-03, -2.70634983e-02, -5.61619072e-06, ...,
          4.52458102e-04,  7.46375043e-03,  1.59790297e-03],
        [-7.00793974e-03,  1.44732222e-02,  2.39749867e-02, ...,
          1.12998430e-02,  7.17586046e-03,  1.64587833e-02]],

[[-2.22248528e-02,  1.37647998e-03,  4.65444149e-03, ...,
         -1.97386928e-02,  9.03771259e-03,  2.50672698e-02],
        [ 9.19027999e-03, -2.68077068e-02,  1.95314530e-02, ...,
          1.54939098e-02, -2.00274736e-02,  2.99740094e-03],
        [ 2.34596301e-02,  1.67632252e-02, -6.27557002e-03, ...,
         -1.08365873e-02, -7.35150184e-03,  6.66996371e-03],
        ...,
        [-1.90700274e-02, -1.38433194e-02, -1.51757000e-03, ...,
          1.32651012e-02, -7.46375322e-03,  1.20992884e-02],
        [-1.88179333e-02, -2.54113376e-02, -3.25070918e-02, ...,
          1.15042981e-02,  8.93889368e-03,  7.20201572e-03],
        [ 5.81856220e-05, -1.47801684e-02, -6.14402071e-03, ...,
          2.19213814e-02, -1.27481399e-02, -9.92694288e-04]],

...,

[[-2.87456717e-02, -1.09022148e-02,  3.72917065e-03, ...,
          2.84757256e-03, -2.41442006e-02, -1.80615962e-03],
        [-2.47977357e-02,  7.77481496e-03,  2.08935030e-02, ...,
         -1.25209223e-02,  1.95152499e-02, -7.82345596e-04],
        [ 2.28727199e-02,  3.62797803e-03, -3.36102932e-03, ...,
         -7.89200247e-04,  1.11413868e-02,  1.42954113e-02],
        ...,
        [ 4.80046915e-03,  1.22773862e-02,  1.52080124e-02, ...,
         -1.30401151e-02,  1.12529064e-03, -4.39310865e-03],
        [-1.11971144e-02, -1.20939268e-03, -5.99930063e-03, ...,
          4.71367650e-02, -8.55424223e-05, -8.41138326e-03],
        [ 9.04783234e-03,  1.26670627e-02,  1.63898908e-03, ...,
         -1.16018904e-03,  3.22799459e-02, -7.18271732e-03]],

[[ 1.08676432e-02,  1.36004072e-02, -4.60197730e-03, ...,
          2.87630036e-02, -2.49077771e-02, -2.05047079e-03],
        [ 9.90082044e-03, -1.59557220e-02, -1.96074508e-03, ...,
          2.09254050e-03, -1.57696120e-02,  3.15909827e-04],
        [-2.35079471e-02, -3.37247271e-03, -1.78314392e-02, ...,
         -1.14336153e-02, -1.68020222e-02, -1.03917411e-02],
        ...,
        [ 1.23332348e-03,  1.11576319e-02, -1.20590627e-02, ...,
          2.07730359e-03, -1.43355839e-02,  1.05062481e-02],
        [ 1.49237653e-02,  7.88312312e-03,  4.65499557e-04, ...,
          2.46078521e-02, -4.54832986e-03,  1.02313124e-02],
        [-2.73424922e-03,  1.83491651e-02,  1.74229196e-03, ...,
         -9.55377705e-03, -5.65235782e-03, -1.14087705e-02]],

[[ 1.09337596e-02, -2.10437980e-02, -1.05800685e-02, ...,
         -6.56963792e-03,  1.45507127e-03,  8.91961996e-03],
        [-3.75897298e-03, -6.11150730e-03,  2.85932445e-03, ...,
          1.28849866e-02,  3.19971680e-03, -5.58700413e-04],
        [ 5.85980422e-04, -4.57300013e-03, -1.03927888e-02, ...,
         -4.13403697e-02, -1.30845634e-02,  1.71099808e-02],
        ...,
        [-2.68593943e-03,  3.69987800e-04,  1.51738245e-03, ...,
          4.83785942e-03,  1.66357215e-02, -5.48439845e-03],
        [ 1.20854368e-02,  5.05305175e-03,  2.46214960e-02, ...,
         -3.43978219e-02,  7.53583247e-03,  3.36864591e-02],
        [-1.13962796e-02, -1.49790356e-02, -1.38309961e-02, ...,
          6.29504793e-04, -1.78377591e-02, -1.49134332e-02]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.5389867 , 0.61479825, 0.60809326, ..., 0.6191155 ,
         0.6289523 , 0.57391906],
        [0.39017102, 0.44472748, 0.4764988 , ..., 0.5190437 ,
         0.38948387, 0.4838077 ],
        [0.5748676 , 0.62614745, 0.5152158 , ..., 0.5696252 ,
         0.43688968, 0.644043  ],
        ...,
        [0.528488  , 0.46211687, 0.5646755 , ..., 0.52592915,
         0.70118403, 0.5618546 ],
        [0.58283436, 0.5550209 , 0.6565793 , ..., 0.7949155 ,
         0.49571222, 0.5169585 ],
        [0.5333895 , 0.62617797, 0.5919402 , ..., 0.6056625 ,
         0.5645018 , 0.6600134 ]]], dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': array([0.14654781, 0.22278094, 0.2743674 , 0.20907548, 0.24956009,
       0.18405011, 0.21791767, 0.22734475, 0.2668076 , 0.20462586,
       0.19914745, 0.28613123, 0.23778196, 0.27146855, 0.23280822,
       0.20081756, 0.19908659, 0.18805924, 0.23253727, 0.18743424,
       0.1948078 , 0.17780925, 0.19240059, 0.20284435, 0.23953158,
       0.2414018 , 0.22483683, 0.27306545, 0.1580696 , 0.23140562,
       0.2652837 , 0.25217816, 0.25966236, 0.21986026, 0.1649964 ,
       0.2737241 , 0.21450056, 0.2052232 , 0.2948621 , 0.2333531 ,
       0.21151926, 0.17604513, 0.20712714, 0.27924615, 0.23490104,
       0.25804925, 0.30132753, 0.25481915, 0.26180637, 0.28384343,
       0.29114008, 0.2278892 , 0.13483137, 0.28901073, 0.22650169,
       0.19220173, 0.20297801, 0.24761178, 0.22846888, 0.22040966,
       0.27271867, 0.31886327, 0.24663903, 0.20238942, 0.21435668,
       0.18714531, 0.23238699, 0.26992947, 0.20557305, 0.18135974,
       0.1717081 , 0.13401309, 0.20160556, 0.24493887, 0.15000689,
       0.24136183, 0.25651586, 0.2202598 , 0.20814568, 0.27010095],
      dtype=float32)
                },
                'post': {
                    'b': array([-0.00229272,  0.00798599,  0.00748841, ..., -0.00560183,
       -0.00854836, -0.00232937], dtype=float32),
                    'w': array([[[-0.00671915,  0.00587748, -0.01147179, ...,  0.01086278,
          0.00020015, -0.0060481 ],
        [ 0.00825962, -0.00934129, -0.00360557, ...,  0.0052317 ,
          0.01242181, -0.00182717],
        [-0.00623296,  0.00317535,  0.00204991, ...,  0.00817461,
          0.00050625,  0.01241348],
        ...,
        [-0.01216628,  0.0058833 ,  0.00036413, ...,  0.00734476,
         -0.00440068, -0.00117811],
        [-0.00370096,  0.00782828,  0.00343848, ..., -0.00323366,
         -0.01976196,  0.01709105],
        [-0.01143755, -0.00773526, -0.03307534, ...,  0.0074515 ,
         -0.01465548, -0.00724028]],

[[ 0.02401702, -0.01838864,  0.00415906, ...,  0.02648286,
          0.00518504, -0.0024082 ],
        [ 0.00661192, -0.02460841, -0.0046568 , ...,  0.00501935,
          0.03585167, -0.01208579],
        [ 0.01364956,  0.00045761, -0.00290023, ...,  0.00389908,
         -0.00546074, -0.01697332],
        ...,
        [-0.0167884 ,  0.01635807, -0.01343242, ..., -0.0044075 ,
         -0.01375843, -0.01890904],
        [ 0.00859275,  0.00427125, -0.00558521, ...,  0.00896568,
         -0.02031408,  0.01403312],
        [-0.00814684, -0.03470473,  0.00091518, ...,  0.02409742,
         -0.02204975, -0.00905873]],

[[-0.02547508, -0.03178998, -0.02492431, ...,  0.01206358,
          0.01815842,  0.01031798],
        [ 0.01094309, -0.01770925, -0.00870426, ..., -0.0442341 ,
          0.01793398,  0.00238465],
        [ 0.02443501, -0.0141692 ,  0.00303102, ..., -0.01523645,
         -0.00715213,  0.01152117],
        ...,
        [-0.00975715, -0.01163363, -0.02921766, ..., -0.00062836,
          0.02164087,  0.00522299],
        [-0.0054695 ,  0.00029186,  0.01624663, ..., -0.00645192,
         -0.02451107,  0.01980308],
        [-0.00294931, -0.02684786,  0.0132309 , ..., -0.02287843,
          0.01756131, -0.00595226]],

...,

[[-0.02918898, -0.00351841, -0.02531841, ...,  0.03394908,
          0.00709433, -0.03074959],
        [ 0.00805127, -0.0107296 , -0.00784526, ...,  0.00062244,
          0.00750641, -0.01364604],
        [ 0.0001563 ,  0.0029194 , -0.00859232, ..., -0.00644335,
          0.00354821, -0.01876928],
        ...,
        [ 0.01682767,  0.00774987,  0.02304111, ...,  0.0160309 ,
         -0.02701204, -0.01488684],
        [ 0.0029405 ,  0.02021392,  0.00494514, ..., -0.00417519,
         -0.03744255,  0.0043031 ],
        [ 0.02299108, -0.01505302, -0.02368749, ..., -0.01873803,
         -0.03611644,  0.00796025]],

[[ 0.0122453 ,  0.01326727,  0.03010734, ..., -0.01486119,
          0.02018904, -0.03737747],
        [-0.02100579,  0.02848019, -0.01392079, ...,  0.00715955,
         -0.00694668,  0.00217169],
        [-0.00659102,  0.00778019,  0.03202567, ...,  0.00394989,
          0.00520773,  0.02136467],
        ...,
        [ 0.00828948,  0.00056415,  0.00498227, ..., -0.01156233,
         -0.0243761 , -0.01975311],
        [-0.0084665 , -0.00707852,  0.01288811, ..., -0.00492141,
          0.01152811, -0.00993543],
        [-0.01454077,  0.00565558, -0.0163257 , ...,  0.03055262,
         -0.02436376, -0.01799387]],

[[ 0.00178173, -0.0117442 , -0.0419663 , ...,  0.00717429,
         -0.0201303 ,  0.02430891],
        [ 0.00307834, -0.02651902, -0.01245268, ..., -0.00345665,
         -0.01771973, -0.00738682],
        [-0.02064985, -0.00220527, -0.0091716 , ..., -0.00358126,
         -0.01540149, -0.0127347 ],
        ...,
        [-0.01497488,  0.00410123,  0.00180891, ...,  0.00390501,
          0.01068672, -0.00476688],
        [ 0.01522674,  0.00994318,  0.02885547, ...,  0.00942743,
          0.03936775,  0.02158558],
        [ 0.01009504,  0.00526136, -0.0112782 , ..., -0.00397819,
         -0.01786978,  0.01006007]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.68155843, 0.62903583, 0.6750482 , ..., 0.6066878 ,
         0.5988386 , 0.61162263],
        [0.62814796, 0.6038018 , 0.37679884, ..., 0.56483936,
         0.6389052 , 0.3288467 ],
        [0.39298034, 0.51051736, 0.54328805, ..., 0.332344  ,
         0.33484176, 0.54041505],
        ...,
        [0.5967134 , 0.45429158, 0.5881717 , ..., 0.6005155 ,
         0.6570745 , 0.4317941 ],
        [0.3347121 , 0.53597665, 0.66715825, ..., 0.53718925,
         0.67430377, 0.5368181 ],
        [0.5099823 , 0.6339526 , 0.66276133, ..., 0.5511503 ,
         0.60982627, 0.59092534]]], dtype=float32)
                },
                'query': {
                    'b': array([[-0.01693531,  0.01281131,  0.00499396, ...,  0.00779316,
        -0.00025738,  0.01801906],
       [-0.02124937,  0.00200343, -0.00883132, ...,  0.00201107,
         0.00554235,  0.01515533],
       [-0.00305617,  0.004093  , -0.0067767 , ...,  0.00522619,
         0.01738874, -0.00898235],
       ...,
       [-0.0055667 , -0.0178722 ,  0.01464478, ...,  0.00780344,
         0.00424719,  0.00944584],
       [ 0.017694  , -0.02612935,  0.0004033 , ..., -0.0020163 ,
         0.01824481,  0.01316419],
       [ 0.02740095, -0.00695968, -0.00573279, ...,  0.00210098,
         0.01201493,  0.00295716]], dtype=float32),
                    'w': array([[[-0.04122668,  0.03792162,  0.01956038, ...,  0.00083074,
         -0.02338985,  0.00243163],
        [-0.00712831,  0.01235137, -0.02554855, ..., -0.01994032,
          0.00118904, -0.00247901],
        [-0.01280304, -0.00637479,  0.02810878, ...,  0.00658902,
          0.03580311, -0.03428695],
        ...,
        [-0.02720779, -0.01454407,  0.02582767, ..., -0.02545251,
         -0.01254367, -0.05030369],
        [-0.0039447 , -0.02432956, -0.00034708, ..., -0.01586003,
          0.01817022,  0.03687435],
        [-0.003418  ,  0.02158768, -0.00758175, ...,  0.00150226,
         -0.04043954, -0.01866934]],

[[-0.03692956,  0.01509409,  0.01384334, ..., -0.00654719,
         -0.00732074,  0.04548213],
        [-0.0038643 ,  0.01272917, -0.00321571, ..., -0.01163196,
          0.00438737,  0.02364455],
        [-0.02802553, -0.03109397, -0.00476508, ..., -0.01181234,
          0.01446376, -0.02256288],
        ...,
        [-0.02395813, -0.04411834,  0.04697748, ..., -0.00915199,
          0.0028583 ,  0.02856146],
        [-0.01343076,  0.02960327,  0.02965372, ..., -0.0018596 ,
          0.02850719, -0.02436874],
        [ 0.00915562,  0.03533962,  0.0076619 , ..., -0.00566451,
         -0.00794141,  0.00355512]],

[[ 0.00529579,  0.01058671,  0.00011963, ..., -0.01461823,
          0.01832964, -0.03549372],
        [-0.02683492, -0.02454132, -0.01372553, ...,  0.04161058,
          0.00430586,  0.00691275],
        [-0.02324759,  0.05466631,  0.00990334, ...,  0.00329311,
          0.02167726,  0.01505934],
        ...,
        [-0.02151913, -0.01149378,  0.03133911, ...,  0.00814379,
          0.01358099,  0.00612202],
        [-0.00223094,  0.00118736,  0.0109336 , ...,  0.00330162,
          0.01970872, -0.01437279],
        [-0.01775653,  0.01499598, -0.01049756, ..., -0.02359271,
          0.00852719, -0.03158678]],

...,

[[-0.04718875, -0.02553122, -0.00076986, ..., -0.00795767,
          0.03631214, -0.03385771],
        [-0.00769815, -0.02347803,  0.0161705 , ..., -0.01839636,
         -0.02384454, -0.00762767],
        [ 0.00525161,  0.03425131, -0.01315107, ..., -0.02119639,
          0.01107599, -0.02556389],
        ...,
        [ 0.00995079,  0.02518961,  0.01715655, ..., -0.02399958,
         -0.00555898, -0.00427775],
        [-0.02353664, -0.04275462, -0.03016347, ...,  0.00946756,
         -0.01889918, -0.00561448],
        [ 0.00777182, -0.00205447, -0.00984231, ...,  0.01325912,
         -0.00156405,  0.01492274]],

[[-0.04279121, -0.01208505, -0.00438044, ...,  0.00992067,
         -0.00284215, -0.02040368],
        [ 0.00699078,  0.00958052,  0.00296794, ..., -0.02966005,
          0.01462817,  0.04479409],
        [ 0.00338127,  0.00096115, -0.0127215 , ...,  0.000855  ,
          0.00223876, -0.00357484],
        ...,
        [-0.00160912,  0.04322622, -0.00449747, ...,  0.01171507,
          0.0244883 ,  0.01934879],
        [-0.03091593,  0.00955663,  0.02056676, ...,  0.00886375,
         -0.04180507, -0.04430168],
        [ 0.00873103, -0.0478724 ,  0.01522835, ..., -0.03086085,
         -0.04521716,  0.02666597]],

[[ 0.01294661, -0.01451382, -0.01001048, ...,  0.03827586,
          0.02376596,  0.00691161],
        [ 0.01033855, -0.01908808, -0.00044221, ..., -0.01828135,
          0.0102315 , -0.0078474 ],
        [ 0.02605218,  0.05498118,  0.04802718, ...,  0.03354565,
         -0.02080743, -0.01015075],
        ...,
        [-0.02980904,  0.01638061, -0.01147437, ..., -0.0347767 ,
          0.0278121 ,  0.00401018],
        [ 0.0189371 , -0.00872775, -0.03812696, ..., -0.02774808,
          0.00812404,  0.01737259],
        [ 0.02776276,  0.01406731, -0.01994073, ...,  0.01034588,
         -0.02899795, -0.01627316]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.9376331 , 0.7032849 , 0.7677844 , ..., 0.6530718 ,
         0.92840385, 0.9566358 ],
        [0.584121  , 0.6078253 , 0.6337089 , ..., 0.74463034,
         0.589317  , 0.67593604],
        [0.7987061 , 0.8131216 , 0.76468116, ..., 0.7553356 ,
         0.80643433, 0.79187274],
        ...,
        [0.9751518 , 0.81484234, 1.0045264 , ..., 0.63805324,
         0.80125165, 0.835067  ],
        [0.841809  , 1.0036488 , 0.80288726, ..., 0.7201403 ,
         0.82165307, 0.93769443],
        [0.9835393 , 0.9208843 , 0.61715674, ..., 0.6475194 ,
         0.84889257, 0.74949217]]], dtype=float32)
                },
                'value': {
                    'b': array([[-0.00188822, -0.00093386,  0.00192111, ...,  0.00245047,
         0.00135479,  0.001175  ],
       [ 0.0006789 ,  0.00084782, -0.00147296, ..., -0.00530237,
         0.00315595, -0.00136973],
       [-0.00260676,  0.00303111, -0.00157738, ..., -0.00982555,
        -0.01084454,  0.00202332],
       ...,
       [-0.0045933 , -0.00203996,  0.00438417, ..., -0.00307284,
        -0.00089111,  0.00201392],
       [-0.0005301 , -0.00109679, -0.0009102 , ..., -0.00083201,
         0.00086958, -0.0019664 ],
       [-0.00357503,  0.00066696,  0.00028513, ...,  0.00167007,
        -0.0029751 , -0.0002654 ]], dtype=float32),
                    'w': array([[[ 7.46658957e-03, -8.41677841e-03,  3.31376195e-02, ...,
         -2.65153358e-03, -3.16191697e-04,  1.41285108e-02],
        [-3.97422947e-02, -1.77706983e-02, -3.90480249e-03, ...,
          1.01006670e-04, -1.42450503e-03, -1.40604619e-02],
        [-1.60975344e-02,  6.63828477e-03,  3.24483099e-03, ...,
         -2.85821017e-02,  1.02422163e-02, -2.28672903e-02],
        ...,
        [-1.64939910e-02,  1.04231611e-02, -3.46118659e-02, ...,
          8.36394448e-03, -2.87051387e-02, -7.90313631e-03],
        [-3.05770291e-03,  4.16853727e-04,  2.16026939e-02, ...,
          1.48915090e-02, -1.02293175e-02,  4.35859896e-03],
        [-6.22540386e-03, -1.09573379e-02, -4.03586263e-03, ...,
          2.01954097e-02,  8.48009065e-03, -1.28198322e-02]],

[[-5.52004483e-03,  1.89961959e-02, -3.90998181e-03, ...,
         -1.90473497e-02,  4.76297783e-03,  2.83387280e-03],
        [ 1.63630396e-02, -4.15390544e-03,  2.08455301e-03, ...,
         -2.42315549e-02,  1.16302306e-02,  1.27262278e-02],
        [-5.75591670e-03, -5.23624616e-03, -1.03481170e-02, ...,
         -2.44012047e-02,  1.49863632e-02,  2.14042440e-02],
        ...,
        [ 3.10197677e-02, -2.56757112e-03, -1.81192881e-04, ...,
          2.98023969e-02, -1.16637945e-02, -8.87844712e-03],
        [-7.10112322e-03,  2.40918100e-02,  2.50787288e-02, ...,
         -4.20905724e-02, -3.14644421e-03,  8.11068341e-03],
        [-1.47604046e-03,  2.19105929e-02,  1.08880852e-03, ...,
          1.83684798e-03,  2.12709773e-02,  3.93415010e-03]],

[[ 1.10501598e-03,  2.73730624e-02,  2.43027713e-02, ...,
          2.16459506e-03, -5.12673892e-03, -1.41830761e-02],
        [-1.81094911e-02,  1.02478201e-02,  8.10321979e-03, ...,
         -9.59546887e-05,  3.78491846e-03, -2.81681237e-03],
        [ 2.04638834e-03, -1.02619280e-03, -1.28681334e-02, ...,
          2.41935141e-02,  4.52202366e-04, -1.22413626e-02],
        ...,
        [ 1.95489693e-02, -1.17230311e-03,  4.46764827e-02, ...,
         -2.36402769e-02, -3.47259716e-04, -7.93880690e-03],
        [-3.81625025e-03, -2.29792725e-02,  2.53481586e-02, ...,
          1.41859790e-02,  8.37325770e-03, -1.75706402e-03],
        [-2.04868615e-03,  3.24926479e-03, -1.33442329e-02, ...,
          4.60843975e-03, -2.01913826e-02,  5.13361255e-03]],

...,

[[-9.99507774e-03, -4.94171446e-03,  1.16261840e-02, ...,
         -1.68386642e-02, -2.04617456e-02,  8.15850869e-03],
        [ 1.51357614e-02, -4.35531046e-03,  1.12676586e-04, ...,
         -3.06487251e-02, -2.48786379e-02,  1.26590207e-02],
        [-1.33918896e-02, -2.56765867e-03,  1.56452437e-03, ...,
          3.90092516e-03, -8.07243632e-04,  1.59494597e-02],
        ...,
        [ 2.47527380e-02,  4.95977700e-04, -7.83556513e-03, ...,
          1.29263969e-02, -1.06329266e-02, -1.36266905e-03],
        [ 1.88716105e-04, -6.92628231e-03,  2.89969845e-03, ...,
          1.57398917e-02,  9.40438733e-03, -1.51683474e-02],
        [-4.69554914e-03,  8.82130768e-03,  3.60536389e-02, ...,
         -1.17771151e-02,  7.62531161e-03, -1.00490991e-02]],

[[ 2.25756201e-03,  2.87068989e-02, -1.22678857e-02, ...,
          4.17798944e-02, -5.02856495e-03,  3.52256671e-02],
        [ 1.58512164e-02, -4.67397757e-02,  2.44544866e-03, ...,
         -2.65001953e-02,  3.46772326e-03, -6.48112269e-03],
        [-5.30846728e-06,  1.73494173e-03, -2.27560811e-02, ...,
         -1.99548304e-02,  5.29782381e-03, -7.82518822e-04],
        ...,
        [-9.33633931e-03,  8.53182096e-03, -8.93332250e-03, ...,
          1.59947556e-02, -2.01418828e-02,  2.33594086e-02],
        [ 1.11352140e-02,  1.32814553e-02, -1.36366943e-02, ...,
          2.25968240e-03, -1.95194278e-02,  1.89713072e-02],
        [ 1.78836472e-02,  8.50709248e-03, -1.48976594e-03, ...,
         -8.17759801e-03,  5.23386896e-02, -1.55792525e-02]],

[[ 9.88735631e-03,  1.84934516e-03,  1.97978858e-02, ...,
         -4.30854387e-04, -7.47772772e-03, -2.07434222e-02],
        [ 1.15765315e-02,  4.83674072e-02,  6.84570847e-03, ...,
          9.09647997e-03, -1.43847680e-02,  2.62443088e-02],
        [ 5.19147981e-03,  1.08308000e-02, -1.10734832e-02, ...,
          8.27481411e-03,  1.91579498e-02,  3.50668430e-02],
        ...,
        [ 3.39808539e-02,  5.29606361e-03, -6.18779194e-03, ...,
         -2.93455273e-02, -1.68223586e-02,  1.95688456e-02],
        [-2.16533802e-02, -1.91696882e-02,  4.10865666e-03, ...,
         -1.37714809e-02, -1.17597580e-02,  1.65862951e-03],
        [ 2.24986691e-02, -3.52053496e-04, -1.82409468e-03, ...,
          1.69674065e-02,  4.27982099e-02,  6.34616241e-03]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.62496966, 0.66488624, 0.6357599 , ..., 0.6134365 ,
         0.55514896, 0.55824465],
        [0.60652316, 0.60698503, 0.35712233, ..., 0.5366914 ,
         0.5985739 , 0.33324313],
        [0.47283477, 0.56796557, 0.58507115, ..., 0.39520392,
         0.3884123 , 0.5279009 ],
        ...,
        [0.5727812 , 0.5122842 , 0.5560297 , ..., 0.5729769 ,
         0.60154873, 0.5028398 ],
        [0.33485147, 0.50765055, 0.591919  , ..., 0.54024583,
         0.5693564 , 0.54215026],
        [0.5549902 , 0.6323646 , 0.648601  , ..., 0.58587265,
         0.59278625, 0.59465283]]], dtype=float32)
                }
            }
        },
        'x_layers_19': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': array([-0.00279087,  0.00553406, -0.00618837, ..., -0.00082947,
       -0.00121525,  0.00695249], dtype=float32)
                    },
                    'linear': {
                        'w': array([[-0.03057937, -0.03449765, -0.00030052, ..., -0.04533616,
        -0.01315903,  0.00346747],
       [-0.0189806 , -0.0139573 , -0.0167984 , ..., -0.01144543,
        -0.03284094, -0.05118792],
       [-0.03204586, -0.01559104, -0.03456049, ..., -0.05110538,
         0.01989403, -0.0404238 ],
       ...,
       [ 0.01285772, -0.03061882,  0.02235553, ..., -0.02296763,
        -0.01331782,  0.03847779],
       [-0.03716642, -0.03083301, -0.02233933, ...,  0.01910252,
        -0.01984283,  0.01339958],
       [-0.01486247, -0.03447136,  0.02612187, ..., -0.00439616,
         0.02521509,  0.00436651]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.0707961, 1.0929537, 1.0180541, ..., 1.4054673, 1.2883608,
        1.0494804]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': array([-0.00071812,  0.00626113,  0.00534002, ..., -0.00520115,
       -0.00256186,  0.00024727], dtype=float32)
                    },
                    'linear': {
                        'w': array([[-0.00638571, -0.03181123,  0.01637862, ..., -0.02804339,
         0.01380809, -0.01930169],
       [ 0.03387563,  0.03299653, -0.0156489 , ...,  0.04582642,
         0.00388483,  0.0091139 ],
       [-0.02489925, -0.0313705 , -0.00089129, ...,  0.03598975,
        -0.01924329,  0.00141226],
       ...,
       [ 0.0071869 , -0.01164675, -0.12252312, ..., -0.04268306,
         0.03268336,  0.02160267],
       [-0.02873901,  0.03516197, -0.00716065, ..., -0.0070508 ,
        -0.02109207,  0.01088641],
       [ 0.03129392,  0.04549291, -0.00161096, ..., -0.02231779,
        -0.00662258, -0.04896937]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.2743644, 1.6168848, 1.8151819, ..., 1.5378774, 1.3753344,
        1.1602323]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': array([-0.00068695, -0.000179  , -0.00298053, ..., -0.00083815,
       -0.00157204,  0.00269903], dtype=float32),
                    'scale': array([0.01983866, 0.03065088, 0.06953451, ..., 0.05586866, 0.07119216,
       0.00991571], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': array([1.12914  , 1.1227869, 1.1314727, ..., 1.0988649, 1.0992268,
       1.0550259], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': array([[ 5.6358091e-05,  8.3575916e-04, -5.3359644e-04, ...,
         4.5242821e-04,  5.4248731e-04,  4.3794292e-04],
       [ 5.3405447e-04, -3.8205500e-04,  1.9125095e-04, ...,
        -1.8305663e-04,  2.3835654e-04, -4.2093513e-04],
       [-1.5286071e-03,  5.1494699e-04, -7.5894699e-04, ...,
        -2.0895094e-04, -1.0857989e-04,  3.8178050e-04],
       ...,
       [-7.7942845e-05, -1.5454808e-03,  1.1539065e-03, ...,
        -1.2183716e-03, -9.4440789e-04, -8.2142715e-04],
       [ 1.9358263e-04, -2.3736360e-03,  1.2290434e-04, ...,
        -7.5245055e-04, -6.6944674e-05, -3.7660662e-03],
       [-1.1276773e-03, -8.6710806e-04,  8.2271406e-04, ...,
        -4.8099627e-04, -4.1263181e-04, -4.7033969e-05]], dtype=float32),
                    'w': array([[[ 1.66858211e-02, -2.23971736e-02,  1.98825193e-03, ...,
         -1.66950971e-02,  2.84126699e-02,  6.86215423e-03],
        [-3.61021198e-02, -5.99008403e-04, -2.17522564e-03, ...,
         -5.36645204e-03, -1.71211548e-02,  7.97853246e-03],
        [-6.02039229e-03, -7.07927393e-03,  1.50340544e-02, ...,
         -1.07782977e-02,  6.55275024e-03,  7.70088658e-03],
        ...,
        [ 3.66256194e-04, -2.85069866e-04, -9.70526133e-03, ...,
          1.36355665e-02, -6.33951742e-03,  7.70825311e-04],
        [-1.41819892e-02,  1.63590889e-02,  2.55743060e-02, ...,
          2.41388604e-02, -5.91733120e-02,  1.40847396e-02],
        [-8.19051638e-03,  7.65746576e-04,  1.45718094e-03, ...,
          1.06700966e-02,  7.19248224e-03,  7.06929830e-04]],

[[ 9.44192335e-03,  1.46624977e-02,  8.47252458e-03, ...,
          2.07918230e-02,  1.26643688e-03, -7.55505404e-03],
        [-2.74923965e-02,  2.47141216e-02,  3.08834086e-03, ...,
          1.48089584e-02,  2.42911205e-02, -4.90549207e-02],
        [-3.66512663e-03, -2.93549430e-02, -1.96302459e-02, ...,
          4.38228599e-04,  2.24047638e-02,  1.21802660e-02],
        ...,
        [ 3.30321724e-03,  2.70132758e-02, -5.50583797e-03, ...,
         -5.83150424e-03,  3.25948335e-02,  1.21745365e-02],
        [ 2.03197692e-02, -3.31652164e-03,  4.77161491e-03, ...,
         -7.36814970e-03, -4.80881669e-02, -3.46971639e-02],
        [-1.72067042e-02,  4.93288925e-03,  7.18023488e-03, ...,
         -5.45862713e-04,  3.05428263e-02,  1.14726676e-02]],

[[ 4.66989633e-03, -1.16173318e-02, -1.14481961e-02, ...,
         -1.42387012e-02,  1.19096134e-02,  1.67787131e-02],
        [ 5.13310870e-03, -2.19654222e-03, -2.20984705e-02, ...,
          2.04729941e-02,  2.09500808e-02, -2.61275354e-03],
        [ 2.73368694e-02, -3.05800531e-02,  9.87775065e-03, ...,
         -5.37307176e-04,  6.30506128e-03,  1.39507661e-02],
        ...,
        [-4.90885554e-03, -2.40781740e-03,  2.64469478e-02, ...,
          6.77102152e-03,  5.54582290e-03, -4.62119654e-03],
        [-2.67972406e-02, -3.56044956e-02,  5.32912388e-02, ...,
          5.52707072e-03, -6.49520606e-02,  3.88988806e-03],
        [-7.94063695e-03, -3.10174393e-04,  7.78336776e-03, ...,
         -6.03403850e-03, -2.05734046e-03,  3.03571927e-03]],

...,

[[-3.24139744e-02, -6.02530269e-03, -7.98900903e-04, ...,
         -1.58375483e-02, -1.34572200e-02,  4.87663131e-03],
        [ 7.33343977e-03, -2.41723433e-02, -1.95799884e-03, ...,
         -1.51795968e-02, -2.00706185e-03, -4.99610882e-03],
        [-2.00723624e-03, -1.18715595e-02,  1.22464402e-02, ...,
         -6.46425178e-03,  3.72579135e-03, -2.57654442e-03],
        ...,
        [ 3.92827019e-03,  1.34410122e-02, -8.96284648e-04, ...,
         -6.92958152e-03,  2.73795761e-02,  8.46259005e-04],
        [-2.84000430e-02,  2.93577407e-02, -6.41751103e-03, ...,
         -4.59109917e-02, -2.86693443e-02, -8.47621076e-03],
        [ 1.60963318e-04,  1.63372385e-03, -6.12810114e-03, ...,
         -1.44142972e-03, -4.58520232e-03, -7.00910517e-04]],

[[-2.20390093e-02, -7.62513652e-03, -6.85018301e-03, ...,
         -3.59854384e-05,  1.72781307e-04, -4.98305308e-03],
        [ 2.65205055e-02, -1.67112891e-02, -1.54574085e-02, ...,
          2.24714335e-02,  1.77529380e-02,  2.46013198e-02],
        [-1.70388333e-02,  6.67828089e-03,  1.18959672e-03, ...,
          9.80904046e-03, -4.28004051e-03, -8.23487062e-03],
        ...,
        [-1.62568986e-02, -1.11033674e-02, -8.63893249e-04, ...,
         -1.88603196e-02, -7.89354648e-03, -7.79468101e-03],
        [ 7.66636617e-03,  1.45504484e-02, -1.56560801e-02, ...,
         -1.11601595e-02,  1.18275825e-02,  1.06610106e-02],
        [ 7.72172632e-03, -9.92267113e-03,  5.72404405e-03, ...,
         -1.11791305e-02, -9.21111740e-03, -1.16633289e-02]],

[[-5.90128312e-03,  7.19144708e-03, -1.62249047e-03, ...,
          7.17088347e-03,  2.48752953e-03, -1.46825088e-03],
        [-3.90324765e-03, -1.09524617e-03, -3.75927798e-03, ...,
         -2.10372284e-02, -3.45567288e-03, -2.17198627e-03],
        [-1.24420477e-02,  8.93982965e-03, -1.07633350e-02, ...,
          8.87843035e-03,  1.86779350e-02, -1.95613485e-02],
        ...,
        [ 5.58171421e-04,  4.86980774e-04, -2.23068683e-03, ...,
          5.12370095e-03,  1.37364306e-02, -3.50908330e-03],
        [-3.38176861e-02,  7.37074669e-03, -1.00086094e-03, ...,
         -5.08754030e-02, -4.78045829e-02, -2.40573566e-02],
        [ 2.48386350e-04,  5.62367728e-03,  9.40514356e-03, ...,
         -4.55174455e-03, -4.08558873e-03,  5.38862892e-04]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.4346102 , 0.47389197, 0.5610871 , ..., 0.49410427,
         0.47207502, 0.42069682],
        [0.79102457, 0.67554444, 0.65893763, ..., 0.6542111 ,
         0.7654547 , 0.632914  ],
        [0.49814656, 0.6458841 , 0.5418607 , ..., 0.489941  ,
         0.63635606, 0.5992193 ],
        ...,
        [0.53818256, 0.5101116 , 0.5716432 , ..., 0.5488566 ,
         0.6102325 , 0.50402397],
        [1.1352353 , 0.755172  , 0.85898775, ..., 0.7581283 ,
         0.84286636, 0.6225877 ],
        [0.50428194, 0.35342225, 0.29662323, ..., 0.29238492,
         0.44865963, 0.32332343]]], dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': array([0.14652345, 0.16987266, 0.09893259, 0.08280052, 0.13477588,
       0.12619452, 0.1363653 , 0.07585543, 0.20202784, 0.08150236,
       0.18895143, 0.20548019, 0.10704212, 0.20818919, 0.17153841,
       0.14148037, 0.1626222 , 0.23672125, 0.1762036 , 0.1408799 ,
       0.2247247 , 0.20127554, 0.11450737, 0.23013239, 0.16911133,
       0.174619  , 0.25150812, 0.08649732, 0.12320526, 0.18035066,
       0.26151782, 0.19074674, 0.18112373, 0.07410798, 0.1610882 ,
       0.18994205, 0.12926589, 0.18631133, 0.1497615 , 0.22104244,
       0.10986016, 0.1717358 , 0.12376273, 0.1916074 , 0.14726362,
       0.21275796, 0.21343718, 0.20106877, 0.13625608, 0.12418237,
       0.12081207, 0.13094804, 0.23173405, 0.17589185, 0.14757948,
       0.16841263, 0.23804408, 0.18274456, 0.18690562, 0.21431428,
       0.22474097, 0.10995922, 0.15971506, 0.13611062, 0.16957259,
       0.17104879, 0.1870232 , 0.17258014, 0.13192719, 0.19396555,
       0.13188697, 0.21852025, 0.16542722, 0.15202987, 0.10554535,
       0.10272837, 0.14247337, 0.1691001 , 0.17632973, 0.11661879],
      dtype=float32)
                },
                'post': {
                    'b': array([-0.00091276,  0.00443273,  0.00454847, ..., -0.00962823,
        0.00059259, -0.0012725 ], dtype=float32),
                    'w': array([[[-7.94719532e-03, -2.65852176e-03,  1.21650063e-02, ...,
          2.65658367e-04, -1.12542706e-02,  2.68825162e-02],
        [-3.55421146e-03, -1.01088062e-02,  1.50485346e-02, ...,
         -9.76439938e-03,  2.94282492e-02, -9.73171555e-03],
        [ 5.18939516e-04,  1.81097053e-02,  6.01419248e-04, ...,
          1.18027185e-03, -7.42555130e-03,  2.54618265e-02],
        ...,
        [-1.65492687e-02,  3.55440169e-03,  2.95281899e-03, ...,
         -2.62755007e-02, -1.22740353e-02, -1.86174165e-03],
        [-3.05727236e-02,  1.08236717e-02, -5.32779563e-03, ...,
         -5.65503817e-03,  2.51735039e-02, -5.72287897e-03],
        [-7.39056617e-03,  3.59133305e-03, -1.02642123e-02, ...,
         -5.22708427e-03,  1.66715204e-03, -1.24209216e-02]],

[[ 1.30807224e-03,  1.16729299e-02, -2.90400698e-03, ...,
          6.85372436e-03, -9.31178965e-03,  2.06069890e-02],
        [ 7.82348588e-03,  1.55056193e-02,  3.09418095e-03, ...,
          5.16731897e-03, -1.05577428e-02,  8.11680406e-03],
        [ 3.61175160e-03,  7.84322619e-03,  1.80907622e-02, ...,
          4.03312780e-03, -3.18023516e-03,  6.59049395e-03],
        ...,
        [-3.56158279e-02, -1.16370628e-02, -1.41219189e-03, ...,
          1.89180300e-02, -2.74467878e-02, -3.47287301e-03],
        [-5.62192453e-03, -8.17278773e-03,  1.12488875e-02, ...,
          1.27939526e-02, -4.82155150e-03, -4.83958935e-03],
        [-2.13530031e-03,  7.50897732e-03,  3.97171918e-03, ...,
         -3.68629885e-03,  4.02071560e-03, -1.53148808e-02]],

[[ 1.87139343e-02, -6.22958411e-03, -1.33403149e-02, ...,
          2.42486373e-02,  7.89504498e-02,  1.19837606e-02],
        [-4.09423141e-03, -8.58765561e-03,  9.54173319e-03, ...,
         -3.32051539e-03, -1.79900173e-02, -7.67582096e-03],
        [-1.85193624e-02, -1.31419115e-02, -3.38196754e-02, ...,
          1.98092107e-02,  2.68894807e-02, -2.89291202e-04],
        ...,
        [ 1.88732706e-02,  2.43437346e-02, -4.93244221e-03, ...,
         -1.27447741e-02,  2.12939046e-02, -2.12207194e-02],
        [ 6.27729436e-03, -2.19979603e-02,  1.39481369e-02, ...,
         -5.80500532e-03,  3.53065841e-02,  1.41362874e-02],
        [-4.85987309e-03,  1.34906545e-02,  1.28116114e-02, ...,
          8.18209071e-03, -9.01572464e-04,  1.46044642e-02]],

...,

[[ 2.78657191e-02, -1.77703947e-02, -3.10044293e-03, ...,
          1.36325601e-02,  3.88951004e-02,  4.45787096e-03],
        [-1.49057284e-02, -4.78805136e-03, -2.47442769e-03, ...,
         -1.02427257e-02, -2.14624824e-03, -2.96035036e-02],
        [-1.77836735e-02, -5.20461239e-02,  1.11592154e-03, ...,
         -2.00426355e-02,  2.10658424e-02, -1.34270871e-02],
        ...,
        [ 5.26694581e-02,  3.51745598e-02, -1.40735507e-02, ...,
          3.75317335e-02,  1.60997536e-03, -2.95431761e-04],
        [ 4.44412380e-02, -1.01542054e-02,  1.20230149e-02, ...,
         -9.44445748e-03, -8.66943796e-04, -4.45778761e-03],
        [ 1.27893984e-02, -1.30494754e-03,  3.83278308e-03, ...,
         -2.27406505e-03,  1.82995722e-02,  8.44622031e-03]],

[[-1.21306377e-02, -3.21692079e-02, -1.15592079e-02, ...,
         -2.51174048e-02,  7.68887112e-03, -2.25833748e-02],
        [ 1.37536535e-02,  1.61684696e-02,  1.34988269e-02, ...,
          6.19661016e-03,  1.72671396e-02, -9.50195082e-03],
        [ 1.26967402e-02,  4.28606197e-02,  1.20335985e-02, ...,
         -7.95347244e-03, -2.33929902e-02,  1.42888604e-02],
        ...,
        [-7.26331072e-03,  8.84807389e-03,  1.31279109e-02, ...,
         -2.05161772e-03, -6.56594429e-03,  1.17974693e-03],
        [-2.69776154e-02, -6.05842914e-04, -9.54382867e-03, ...,
          3.14884889e-03,  1.67328063e-02,  5.35101769e-03],
        [-8.55931267e-03, -2.02375222e-02,  9.60149337e-03, ...,
          2.67597800e-03,  8.99885222e-03, -1.50756037e-03]],

[[-6.95926370e-03, -3.42211360e-03,  4.11564577e-03, ...,
          2.65010138e-04, -1.12122754e-02,  5.38297603e-03],
        [-4.65994468e-03, -1.20928045e-02,  1.39921932e-02, ...,
         -1.27126719e-03, -1.89517136e-03,  2.80775828e-03],
        [ 9.77591379e-04, -5.88644017e-03,  1.36731090e-02, ...,
          2.15155841e-03, -8.99187071e-05, -9.61266831e-03],
        ...,
        [ 1.08055631e-03, -3.01486365e-02, -7.61171943e-03, ...,
         -2.29782425e-03, -6.79991953e-03,  2.08318476e-02],
        [-1.46988519e-02, -1.81029423e-03,  3.89482989e-03, ...,
         -5.74233290e-03, -2.33056471e-02, -5.32351527e-03],
        [ 1.11967605e-02,  1.63350888e-02, -8.62249546e-03, ...,
          1.04464474e-03, -1.59137808e-02, -5.87823056e-03]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.49137467, 0.5165331 , 0.40487435, ..., 0.57614166,
         0.7385711 , 0.47208744],
        [0.55653554, 0.5085253 , 0.52160275, ..., 0.43052793,
         0.58328766, 0.5769789 ],
        [0.5251519 , 0.79706544, 0.66159356, ..., 0.47279698,
         0.46287674, 0.6284214 ],
        ...,
        [0.694468  , 0.80210054, 0.62862176, ..., 0.5574472 ,
         0.8837489 , 0.6577479 ],
        [0.53318185, 0.59660494, 0.40246564, ..., 0.41857824,
         0.72678834, 0.56148314],
        [0.41620752, 0.37567958, 0.4109969 , ..., 0.3442809 ,
         0.46163717, 0.35714793]]], dtype=float32)
                },
                'query': {
                    'b': array([[-2.3320248e-02, -1.1966302e-02, -2.0297846e-02, ...,
        -2.1983229e-02, -1.9539420e-02,  2.2182582e-02],
       [-5.8301748e-03,  8.1160599e-03, -6.9336863e-03, ...,
        -1.3168788e-02, -8.9469191e-05,  1.0755450e-02],
       [ 2.1333396e-02,  5.7952716e-03,  1.2760594e-03, ...,
         1.3478556e-02, -4.5719063e-03,  8.1177354e-03],
       ...,
       [-1.4207000e-03,  1.6074298e-02, -1.4178736e-02, ...,
         1.6082305e-02,  4.1668117e-03,  2.2911452e-02],
       [-7.9755876e-03, -1.5447275e-02,  1.0383511e-02, ...,
        -1.7911499e-02,  3.1178708e-03, -1.7663836e-02],
       [ 6.2362943e-03,  1.1733231e-02, -1.5588548e-02, ...,
         9.6454900e-03,  6.5174123e-04,  5.2759107e-03]], dtype=float32),
                    'w': array([[[ 0.01879046,  0.00627272,  0.01409843, ...,  0.02526905,
         -0.00265712,  0.05618662],
        [-0.02516003, -0.00326343,  0.04162131, ...,  0.00678667,
          0.01605123, -0.00443892],
        [ 0.02505029,  0.02248173, -0.01596485, ...,  0.00209537,
         -0.03365821,  0.05305671],
        ...,
        [ 0.01634249, -0.00203726, -0.03275561, ...,  0.00196546,
          0.0316908 ,  0.03987049],
        [-0.02480521, -0.0194451 ,  0.03565983, ...,  0.0883149 ,
          0.03553959, -0.0234833 ],
        [ 0.01779898,  0.00122205,  0.00608964, ..., -0.0020426 ,
         -0.01925553, -0.00076039]],

[[ 0.00652799, -0.02839901, -0.02741853, ...,  0.02932008,
         -0.00974062,  0.02611788],
        [-0.00210414,  0.0219893 ,  0.00270763, ..., -0.01118749,
          0.00783939, -0.01056911],
        [ 0.03691104,  0.00079352, -0.01210357, ...,  0.0437585 ,
          0.02423981,  0.01440724],
        ...,
        [ 0.00354679,  0.00251235, -0.017914  , ...,  0.0125197 ,
         -0.02680792,  0.02465376],
        [-0.04364711,  0.05041509, -0.00353731, ...,  0.03434016,
          0.01251504, -0.04037841],
        [ 0.03486454, -0.00646203,  0.00676579, ..., -0.00762064,
         -0.00691804, -0.00423464]],

[[ 0.00020106, -0.04489181,  0.02027986, ..., -0.00491442,
          0.00433424,  0.02050966],
        [ 0.00396067,  0.02436833, -0.0161712 , ..., -0.03914164,
          0.0173782 ,  0.01984163],
        [-0.00406685,  0.00925299,  0.03284234, ..., -0.00530171,
         -0.0209123 ,  0.02223127],
        ...,
        [ 0.00526062,  0.03031921,  0.05909203, ...,  0.01941052,
          0.01949914,  0.01225612],
        [-0.008929  , -0.00128671, -0.00466132, ..., -0.02247646,
          0.0257553 , -0.04272794],
        [-0.02139043, -0.00907594,  0.00769528, ...,  0.00029861,
          0.00117301, -0.00809446]],

...,

[[ 0.00253991, -0.03459919, -0.03472043, ..., -0.02788465,
         -0.05178779,  0.00252121],
        [ 0.01394676,  0.00404595, -0.00634012, ..., -0.02650934,
          0.02203804,  0.03976254],
        [-0.01617502, -0.07627665, -0.00836789, ..., -0.00036758,
         -0.04927359,  0.01272027],
        ...,
        [ 0.01325031,  0.02337383,  0.036486  , ..., -0.01558252,
          0.06598817,  0.04402451],
        [ 0.04530564, -0.00504782, -0.05810281, ..., -0.04396772,
          0.04688365,  0.04765055],
        [-0.01750285, -0.0137295 , -0.0008545 , ..., -0.01866446,
         -0.01201091, -0.01511886]],

[[-0.00161704,  0.0636497 ,  0.03310066, ...,  0.03317278,
         -0.00975784,  0.00203616],
        [-0.00784977, -0.00143608, -0.00119261, ..., -0.00969103,
         -0.00284597, -0.00950043],
        [-0.0133302 ,  0.00449178,  0.0170726 , ..., -0.01746928,
         -0.01152161, -0.02701238],
        ...,
        [ 0.03547777,  0.00517237,  0.03276812, ..., -0.01695102,
         -0.01511849, -0.01049042],
        [-0.02369291,  0.06456988,  0.04729669, ...,  0.03509759,
         -0.01386307,  0.05301804],
        [ 0.00068137,  0.01429459,  0.01626539, ..., -0.0097482 ,
          0.00330162,  0.00138675]],

[[-0.01105381,  0.00393943, -0.01289818, ..., -0.04812491,
          0.01336343, -0.01491917],
        [ 0.00134544,  0.0092295 , -0.0430508 , ..., -0.01045833,
          0.00656331,  0.00142347],
        [ 0.01488701, -0.03062654, -0.0072165 , ..., -0.02095346,
          0.01998034, -0.00554285],
        ...,
        [ 0.02381367, -0.01228064, -0.00435461, ..., -0.0181347 ,
          0.05034123, -0.00717285],
        [ 0.01958305, -0.04317761, -0.02946939, ...,  0.00058659,
          0.01312297,  0.0187073 ],
        [-0.02160233,  0.01526545,  0.01430138, ..., -0.00187082,
          0.02324795,  0.00513071]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.6408819 , 0.84640783, 0.8163637 , ..., 0.9518507 ,
         0.888962  , 0.8690987 ],
        [0.73040235, 0.7736111 , 0.80893993, ..., 1.0213952 ,
         0.74447936, 0.72983015],
        [0.76767474, 0.783122  , 0.69143456, ..., 0.65028834,
         0.81128454, 0.8034697 ],
        ...,
        [0.75814897, 0.7847155 , 0.9237858 , ..., 0.7149289 ,
         0.90169346, 0.7636707 ],
        [0.8381301 , 1.1863531 , 1.1403961 , ..., 1.2748408 ,
         1.116608  , 1.3532289 ],
        [0.54481393, 0.46388093, 0.32996354, ..., 0.36607277,
         0.57823944, 0.38903984]]], dtype=float32)
                },
                'value': {
                    'b': array([[ 2.6284866e-03,  2.7871240e-04, -6.0292310e-03, ...,
        -7.3629955e-04,  1.2970131e-03, -3.3789459e-03],
       [-1.7902206e-03, -1.6401059e-03, -3.2759264e-05, ...,
        -6.9711115e-03, -1.6303200e-03,  2.0977818e-04],
       [-4.6065627e-05, -1.3298161e-03,  1.7794441e-03, ...,
         1.8877160e-03,  1.4194903e-03, -3.3245850e-03],
       ...,
       [-2.9255324e-03, -3.5976777e-03,  2.1950365e-03, ...,
         8.8367925e-04, -2.0187130e-05,  7.3943031e-04],
       [-2.8092548e-04,  1.5679412e-03,  1.5219685e-03, ...,
        -1.4931384e-03, -9.9890260e-04,  2.8214464e-03],
       [ 4.5016697e-03, -4.0114883e-04, -4.4532988e-04, ...,
         1.4430106e-04,  5.1092445e-03, -6.9501926e-04]], dtype=float32),
                    'w': array([[[ 9.36927367e-03,  2.24846452e-02,  2.45292974e-03, ...,
         -8.28975625e-03, -2.75090914e-02,  9.59235243e-03],
        [ 1.71270373e-03, -1.39735130e-04,  3.37577946e-02, ...,
         -4.37170640e-03, -3.14194057e-03, -2.15227436e-02],
        [-1.54672004e-03, -2.74255257e-02, -3.03714871e-02, ...,
          1.52025865e-02,  1.13139674e-03, -4.77866316e-03],
        ...,
        [ 3.19389775e-02, -5.01666870e-03, -1.28704822e-02, ...,
          5.77493198e-03,  8.06158991e-04, -5.89923875e-04],
        [ 3.38947400e-02, -1.96365695e-02, -1.01442384e-02, ...,
          5.47762588e-03, -2.87755374e-02, -2.06911899e-02],
        [ 2.48693340e-02,  5.00555011e-03,  7.18526216e-03, ...,
          2.34933128e-03,  9.19876806e-03,  3.31273419e-03]],

[[ 9.80129931e-03,  2.17099488e-02,  2.95260048e-04, ...,
         -3.82273607e-02, -1.41924969e-03,  1.57759115e-02],
        [-2.64184992e-03, -3.29675130e-03,  2.89534626e-04, ...,
         -1.40790548e-02,  2.31006183e-02, -1.53553598e-02],
        [-3.41082900e-03,  1.19396141e-02, -1.58081092e-02, ...,
         -7.00374506e-03, -8.09233263e-03,  3.76101360e-02],
        ...,
        [ 4.00474183e-02,  1.74595453e-02,  7.58717582e-03, ...,
         -3.46433409e-02,  3.96660380e-02, -5.45665761e-03],
        [ 1.88549981e-03, -1.83171798e-02,  1.66438826e-04, ...,
          7.38240127e-03, -1.60513353e-02, -2.46273503e-02],
        [ 2.00332943e-02,  1.38896154e-02, -2.72259582e-03, ...,
          6.19711028e-03,  1.73531491e-02,  9.71757993e-03]],

[[ 2.28530057e-02, -3.43316351e-03, -1.09825190e-02, ...,
          1.86314678e-03, -2.29711384e-02, -1.13402272e-03],
        [ 2.73919590e-02,  7.63438269e-03, -1.38108013e-02, ...,
         -8.52864049e-03, -1.38992714e-02, -5.12175448e-03],
        [-1.29722245e-02,  1.97747592e-02,  3.26783545e-02, ...,
         -1.27249137e-02, -2.11906154e-02,  6.62280107e-03],
        ...,
        [-5.70151396e-03,  1.05547793e-02,  3.54975066e-03, ...,
         -1.73960323e-03, -7.88255013e-04, -4.23239311e-03],
        [ 1.30904827e-03, -5.83353546e-03,  5.43385418e-03, ...,
          5.31763816e-03, -1.75985917e-02,  1.77256502e-02],
        [ 5.70553541e-03,  7.49326590e-03, -2.25353986e-02, ...,
          4.40458441e-03, -7.06102094e-03, -4.03826758e-02]],

...,

[[-1.77575145e-02, -9.43127926e-03, -8.28685123e-04, ...,
          7.44167264e-05,  2.48011760e-03,  2.45832559e-02],
        [-9.55136726e-04, -1.74930561e-02, -7.24047562e-03, ...,
         -1.53615223e-02, -6.47678331e-04,  1.55877527e-02],
        [ 9.09104757e-03,  4.13396955e-02, -1.36180939e-02, ...,
          8.72317702e-03, -2.31131576e-02,  2.40340289e-02],
        ...,
        [-3.59102860e-02, -5.37756272e-02,  1.28643000e-02, ...,
         -2.71502603e-02,  3.60584632e-03,  1.79227837e-03],
        [ 6.88599190e-03,  1.01979347e-02, -2.75513642e-02, ...,
          8.40011239e-03,  5.23199029e-02, -6.37135794e-03],
        [ 5.07814391e-03, -1.25527168e-02,  6.37774123e-03, ...,
          4.62807482e-03,  1.63983740e-02,  8.17113928e-03]],

[[-1.06140021e-02,  3.32495407e-03,  1.18820043e-02, ...,
          6.00433256e-03,  3.84307653e-02,  3.38349870e-04],
        [-2.25073323e-02,  2.42929347e-02,  2.67406902e-03, ...,
         -1.93729401e-02,  2.10732073e-02, -3.76919634e-03],
        [-9.21514723e-03, -1.91864427e-02,  1.61433616e-03, ...,
          2.09032781e-02,  4.42184974e-05, -2.94305813e-02],
        ...,
        [ 3.27546895e-02,  9.16469097e-03, -2.50230711e-02, ...,
         -1.99774988e-02, -1.47759747e-02,  1.04548950e-02],
        [ 2.80018197e-03, -1.55748092e-02,  2.01276578e-02, ...,
          3.22249415e-03, -3.75226550e-02, -8.16959795e-03],
        [ 1.80748329e-02,  1.43178338e-02,  4.44830814e-03, ...,
          1.22797107e-02, -7.20503228e-03,  2.02048738e-02]],

[[ 6.23098109e-03,  1.28009240e-03,  1.90864019e-02, ...,
          3.05063510e-03, -1.20336991e-02, -7.22211367e-03],
        [ 2.42555980e-03,  1.72286611e-02, -5.33142500e-03, ...,
         -6.52101682e-03, -1.71236970e-04, -2.13873480e-03],
        [-1.23704579e-02,  3.44731025e-02, -8.13399721e-03, ...,
         -1.78678595e-02,  1.77270956e-02,  1.44108320e-02],
        ...,
        [-6.30496955e-03,  4.70453911e-02,  9.60909482e-03, ...,
          1.36679364e-02,  2.62902980e-03, -2.45311093e-02],
        [-6.66056061e-03,  1.53017507e-04, -3.15158395e-04, ...,
          1.28758838e-02,  3.71902529e-03,  2.27856319e-02],
        [ 3.69637413e-03, -2.70420294e-02,  2.05683662e-03, ...,
         -1.24421576e-03, -3.74466367e-03, -8.87324940e-03]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.52067494, 0.4602856 , 0.40731087, ..., 0.5692328 ,
         0.67637986, 0.45492288],
        [0.5372845 , 0.5612957 , 0.5535549 , ..., 0.48691946,
         0.649278  , 0.5705836 ],
        [0.50430167, 0.7703766 , 0.70108515, ..., 0.5205898 ,
         0.5700291 , 0.7947836 ],
        ...,
        [0.6631088 , 0.79997385, 0.63330853, ..., 0.602523  ,
         0.80915755, 0.65165776],
        [0.48365128, 0.55658865, 0.38854197, ..., 0.42478177,
         0.61644876, 0.4955461 ],
        [0.4433365 , 0.40078905, 0.44940308, ..., 0.37492546,
         0.45279822, 0.48659572]]], dtype=float32)
                }
            }
        },
        'x_layers_2': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': array([-0.00111433,  0.00108239,  0.00224435, ..., -0.00440178,
       -0.00391166,  0.00544218], dtype=float32)
                    },
                    'linear': {
                        'w': array([[ 0.01403686, -0.05290033,  0.01835991, ...,  0.03456852,
         0.04602428, -0.03192553],
       [-0.02005844, -0.00173637, -0.0569217 , ..., -0.02069917,
        -0.02696633, -0.03619819],
       [-0.04991308, -0.00050655,  0.02636448, ...,  0.00382697,
         0.01514277,  0.03238337],
       ...,
       [-0.01799294, -0.02605661, -0.05084125, ...,  0.0214364 ,
        -0.01124236,  0.03825921],
       [ 0.01654951,  0.0341349 ,  0.00021514, ...,  0.05851978,
         0.00840839, -0.01794812],
       [ 0.01197937,  0.00011841, -0.00112626, ..., -0.03313017,
         0.06482383,  0.03593734]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[0.97674125, 1.0421035 , 1.0659041 , ..., 1.0277737 , 0.9845023 ,
        1.0406002 ]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': array([ 0.00058029, -0.00185688, -0.00133239, ...,  0.00196204,
       -0.00302488, -0.00129099], dtype=float32)
                    },
                    'linear': {
                        'w': array([[-0.03089221, -0.02764584,  0.00469999, ...,  0.03527471,
        -0.02542556, -0.03442214],
       [ 0.0074438 ,  0.04577511, -0.00429758, ..., -0.03024394,
        -0.01198478,  0.03910391],
       [ 0.00705164,  0.0343813 , -0.00202623, ..., -0.00482008,
         0.04574559, -0.05240228],
       ...,
       [ 0.00302984, -0.00150592, -0.02762782, ..., -0.04766212,
        -0.01528054,  0.06070133],
       [-0.04403605, -0.02750993, -0.03400619, ...,  0.00910747,
         0.0308539 , -0.01064597],
       [ 0.00639371,  0.04002564, -0.02321222, ...,  0.03084261,
        -0.0078567 , -0.02540698]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.1031147, 1.117435 , 1.0721824, ..., 1.0805465, 1.0726163,
        1.074463 ]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': array([-2.3114951e-03,  6.2909522e-03, -1.1552567e-05, ...,
       -6.4255311e-03, -2.9084997e-03, -8.2080066e-03], dtype=float32),
                    'scale': array([-0.0221999 , -0.00512881, -0.01245704, ..., -0.01569215,
        0.00904967,  0.00978315], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': array([1.1226792 , 1.1108161 , 1.0559344 , ..., 0.97350955, 1.0633241 ,
       1.048979  ], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': array([[ 0.00233258,  0.00309351, -0.00112892, ..., -0.00462134,
        -0.00229245, -0.00666805],
       [-0.00116137,  0.00166665,  0.00024842, ...,  0.000721  ,
         0.00195691,  0.00077325],
       [ 0.00297723,  0.00149911, -0.00517921, ..., -0.00558017,
        -0.00180795,  0.0026498 ],
       ...,
       [-0.00093223, -0.00077007, -0.00018508, ..., -0.00148094,
         0.00050842,  0.0013452 ],
       [-0.00059397, -0.00323179, -0.00278245, ..., -0.00027348,
        -0.00069534,  0.0014734 ],
       [ 0.00202339, -0.00099161,  0.00069069, ..., -0.0009583 ,
         0.00134222, -0.00203341]], dtype=float32),
                    'w': array([[[-8.94103758e-03, -1.66658056e-03,  7.67345587e-03, ...,
         -1.61238182e-02,  8.82412028e-03,  6.46348984e-04],
        [ 8.52084626e-03,  6.26490032e-03,  2.17809398e-02, ...,
          2.06468999e-02, -2.45742756e-03, -1.87717024e-02],
        [ 6.54218346e-03, -3.88751812e-02, -1.36708850e-02, ...,
         -1.14361970e-02, -1.76829230e-02,  3.91271226e-02],
        ...,
        [ 1.93273984e-02,  9.61519405e-03,  6.82339817e-03, ...,
         -2.85228714e-02,  9.51640680e-03, -1.33912545e-02],
        [ 2.96449251e-02,  1.76546443e-02, -8.91647674e-03, ...,
         -1.33323688e-02, -1.52229704e-03, -6.13695988e-03],
        [-7.29722390e-03,  1.29208295e-02, -5.45749702e-02, ...,
          2.03211624e-02,  1.15019334e-02,  4.96047409e-03]],

[[ 1.26578892e-03,  1.09503334e-02,  4.53231577e-03, ...,
         -1.31145259e-02, -2.33688317e-02,  4.28625099e-05],
        [-5.55754174e-03,  1.62886251e-02,  5.64822601e-03, ...,
          1.13027487e-02,  7.36242684e-04,  3.24362935e-03],
        [-9.68135241e-03, -7.98302330e-03,  4.68283240e-03, ...,
         -4.49836487e-03,  6.83220057e-03,  1.51462127e-02],
        ...,
        [ 9.94221400e-03,  1.26088886e-02,  2.28641019e-03, ...,
         -3.45191918e-02,  7.38092978e-03, -1.41630899e-02],
        [ 1.87340043e-02,  4.18769289e-03,  4.22672043e-03, ...,
         -2.26769373e-02, -2.71937959e-02,  2.20513809e-03],
        [ 1.12866051e-02,  9.48542356e-03, -3.27860303e-02, ...,
          2.40716711e-02,  1.24116074e-02, -8.08058027e-03]],

[[ 1.15389684e-02, -5.10842586e-03,  1.51884127e-02, ...,
         -2.44391765e-02, -2.73971353e-02, -1.62927397e-02],
        [ 1.81995286e-03,  2.17429106e-03,  2.88932789e-02, ...,
          2.99599837e-03, -8.92431848e-03, -3.50462529e-03],
        [-9.75059718e-03, -1.89324562e-02, -8.71421210e-03, ...,
          4.71187115e-04, -1.74115680e-03,  8.05928186e-03],
        ...,
        [ 8.22316017e-03,  2.55753081e-02, -4.46758652e-03, ...,
         -2.41416991e-02,  1.29891494e-02, -3.59738097e-02],
        [ 2.58793198e-02,  2.07082555e-02, -8.36309325e-03, ...,
         -1.06869098e-02, -1.78145394e-02,  2.68943328e-02],
        [ 4.88713523e-03,  9.46810283e-03, -5.55097051e-02, ...,
          3.00503429e-02,  8.75395536e-03, -3.49885109e-03]],

...,

[[-6.27226604e-04,  3.18777910e-03,  4.54813568e-03, ...,
         -3.63795290e-04,  1.82062406e-02, -9.44266503e-05],
        [ 1.41483136e-02, -1.42977824e-02, -1.98744275e-02, ...,
         -5.59291197e-03, -1.49107641e-02,  1.43904863e-02],
        [-1.44466478e-02,  2.31337808e-02,  1.27914054e-02, ...,
          1.04759112e-02,  1.75788172e-03, -1.79893188e-02],
        ...,
        [-8.57542641e-03,  1.90665509e-04,  1.82147641e-02, ...,
         -9.12444387e-03,  1.22348024e-02,  2.29855854e-04],
        [-8.97650979e-03, -1.36382217e-02, -1.80628127e-03, ...,
         -7.70106446e-03, -1.71360560e-02, -1.05085205e-02],
        [ 1.78157445e-02, -1.33543545e-02, -5.61761111e-03, ...,
         -5.27891144e-03,  5.65976836e-03, -1.06884856e-02]],

[[-2.75685755e-03,  1.11493738e-02,  1.47611145e-02, ...,
         -2.19679729e-04, -6.11073617e-03, -3.96800460e-03],
        [ 5.37412846e-03,  6.32676808e-03, -1.54358614e-02, ...,
         -1.08150747e-02,  1.65665266e-03,  9.50028421e-04],
        [-8.17670580e-03, -2.11166451e-03, -6.40470535e-03, ...,
         -1.27421115e-02, -1.09132286e-03, -1.41474465e-03],
        ...,
        [ 8.70644674e-03, -2.42831977e-03,  1.19384266e-02, ...,
          2.57595582e-03, -2.19030865e-02, -3.25883669e-03],
        [ 1.43356463e-02,  7.93347321e-03,  1.45333279e-02, ...,
         -2.28662230e-02, -5.29858982e-03,  4.49661072e-03],
        [-2.29433225e-03,  1.86546315e-02,  9.76203568e-03, ...,
         -2.48352438e-02,  1.16549374e-03,  1.17275063e-02]],

[[-4.51580714e-03,  3.84162646e-03,  1.09713534e-02, ...,
          2.67644849e-04, -1.08704967e-02,  1.14232721e-02],
        [ 4.71198326e-03, -4.28085541e-03, -3.10961213e-02, ...,
          1.92970410e-02, -2.13259384e-02, -1.11752655e-02],
        [ 5.23858657e-03,  1.51326796e-02, -6.44034473e-03, ...,
         -1.95504352e-02,  1.60150714e-02,  5.29391272e-03],
        ...,
        [ 1.52046224e-02, -1.51269860e-03, -1.85855143e-02, ...,
          4.54664137e-03, -1.46121706e-03,  1.15908124e-02],
        [-6.32000528e-03,  2.31103622e-03, -7.22707063e-03, ...,
         -8.15302972e-03, -7.59101938e-03,  5.36400476e-04],
        [-1.05950972e-02, -1.67034008e-02, -6.69758208e-03, ...,
         -2.19507534e-02, -1.03131877e-02, -1.33801838e-02]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.43050972, 0.36344343, 0.35904276, ..., 0.3755023 ,
         0.447415  , 0.38662305],
        [0.4069098 , 0.39731088, 0.45042288, ..., 0.41306242,
         0.46482936, 0.40715495],
        [0.36723074, 0.4475068 , 0.32033476, ..., 0.33077934,
         0.43708044, 0.3890885 ],
        ...,
        [0.3949305 , 0.3745144 , 0.36675343, ..., 0.3698377 ,
         0.41757995, 0.34616736],
        [0.42559117, 0.3678387 , 0.39627957, ..., 0.44620186,
         0.43151405, 0.53925216],
        [0.39945847, 0.3871925 , 0.5647749 , ..., 0.5671272 ,
         0.3969689 , 0.38988718]]], dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': array([0.07606351, 0.09519099, 0.08543134, 0.07938089, 0.0478035 ,
       0.05220166, 0.06712113, 0.09687654, 0.07199746, 0.07111655,
       0.09450778, 0.07551523, 0.09265774, 0.08629529, 0.06811125,
       0.07595345, 0.05623099, 0.10769361, 0.07316846, 0.0918607 ,
       0.06741217, 0.06795789, 0.07176007, 0.11473613, 0.06057035,
       0.09941982, 0.10887659, 0.06334271, 0.07600556, 0.11524919,
       0.06202571, 0.0708086 , 0.08118585, 0.06687053, 0.07179871,
       0.07070601, 0.08140425, 0.05987317, 0.06119032, 0.06488035,
       0.08756668, 0.06965479, 0.08641811, 0.07494207, 0.04722185,
       0.10145788, 0.05890552, 0.07864748, 0.06123484, 0.08199554,
       0.10383144, 0.07069265, 0.07977193, 0.08119627, 0.07907593,
       0.06888696, 0.05957579, 0.08070772, 0.06117062, 0.08973919,
       0.06759793, 0.06796879, 0.07075723, 0.08978229, 0.09884267,
       0.06895956, 0.08543922, 0.05556408, 0.08464553, 0.09942926,
       0.07774689, 0.10355261, 0.05608208, 0.09083615, 0.07849273,
       0.06128662, 0.06015168, 0.07853277, 0.06790688, 0.0867492 ],
      dtype=float32)
                },
                'post': {
                    'b': array([-0.00012291,  0.0009653 ,  0.00115237, ...,  0.0021399 ,
       -0.00306339, -0.0042207 ], dtype=float32),
                    'w': array([[[-2.10217144e-02, -3.62480315e-03,  3.03629786e-04, ...,
         -2.47539440e-03, -1.93413980e-02,  3.34993005e-03],
        [-6.61893794e-03,  3.10939061e-03,  9.66002047e-03, ...,
         -1.17761467e-03, -8.80339276e-03, -8.20292626e-03],
        [ 6.34120964e-03, -1.48878177e-03, -2.88486597e-03, ...,
         -4.09498299e-03,  1.14559764e-02, -6.91710680e-04],
        ...,
        [-1.47049502e-02, -1.91921054e-03,  8.64934456e-03, ...,
          1.05494987e-02,  1.48106785e-02, -1.19451089e-02],
        [-9.74751194e-04,  2.43542204e-03,  2.81845173e-03, ...,
          1.61743090e-02, -1.79711059e-02, -7.46698212e-03],
        [-1.45473406e-02, -6.92996383e-03, -2.45343242e-02, ...,
          2.12191343e-02,  5.92035428e-03,  2.15310678e-02]],

[[-3.02073872e-03, -1.19447066e-02,  6.95830071e-03, ...,
          8.04599316e-04, -1.85234323e-02,  2.73672165e-03],
        [ 1.61610811e-03, -1.51668405e-02,  1.11785671e-02, ...,
          7.99397379e-03,  6.24049408e-03, -7.18727987e-03],
        [-4.43224970e-04, -1.43281324e-02, -6.81125233e-03, ...,
         -1.43528488e-02,  1.53827118e-02, -9.75779537e-03],
        ...,
        [-1.34480307e-02, -9.83289909e-03,  1.39178922e-02, ...,
         -4.15747333e-03,  1.47477938e-02, -1.31555833e-02],
        [-1.55448979e-02, -4.96065804e-05,  6.19057752e-03, ...,
          5.98387793e-04, -6.80579385e-03,  1.59067828e-02],
        [-1.11427726e-02, -9.53395292e-03, -7.56308483e-03, ...,
          5.80245350e-03, -1.57048460e-02, -3.80241754e-03]],

[[-1.05666080e-02, -3.12144984e-03,  1.26368692e-02, ...,
          8.27722531e-03, -1.38422092e-02,  1.10680927e-02],
        [ 1.45669142e-03, -1.74943544e-02,  5.36167435e-03, ...,
         -7.89203681e-03, -1.78547837e-02, -5.56232082e-03],
        [-6.58980524e-03,  6.66953996e-03, -8.27248115e-03, ...,
         -1.79896143e-03,  5.58524486e-03,  8.21597001e-04],
        ...,
        [-6.30070514e-04, -1.09762303e-03, -5.81218395e-03, ...,
          6.38545491e-03,  6.11918978e-03,  1.82487536e-03],
        [ 7.82600371e-04, -5.07774577e-03,  4.39119106e-03, ...,
         -7.66993361e-03, -9.38756950e-03,  1.15520600e-02],
        [-9.82622989e-03,  1.02820145e-02, -4.07670662e-02, ...,
          8.49217456e-03, -1.27799343e-02,  9.37399548e-03]],

...,

[[-2.14658980e-03, -1.24297664e-03, -5.49303554e-03, ...,
         -7.40667200e-03,  1.54395960e-03, -7.74664339e-03],
        [-1.86550617e-02, -4.34802845e-03,  1.32464431e-02, ...,
          3.18848435e-03, -5.44274272e-03,  1.85276533e-03],
        [ 9.18497518e-03, -2.09817605e-04, -1.08914764e-03, ...,
         -4.96961782e-03,  1.66279729e-02,  8.12881161e-03],
        ...,
        [ 1.36898621e-03, -5.00992499e-03,  3.27945221e-03, ...,
         -6.60969969e-03, -8.28962307e-03,  6.72791712e-03],
        [-1.07876805e-03, -1.58830751e-02, -6.39187405e-03, ...,
         -9.56065021e-03,  1.23424632e-02, -7.19982339e-03],
        [-1.00100075e-03,  1.47802744e-03, -2.86653196e-03, ...,
         -6.35822816e-03, -8.73123016e-03, -8.55560787e-03]],

[[ 1.25962298e-03, -1.87327880e-02, -2.12054998e-02, ...,
          3.80623760e-03,  5.30398823e-03, -3.95206781e-03],
        [-6.59393147e-03, -9.75189265e-03,  1.15071414e-02, ...,
         -3.19608109e-04,  6.65926328e-03, -1.45214144e-02],
        [ 6.76464243e-03,  1.84686072e-02,  9.12891421e-03, ...,
          2.19555129e-03, -1.74800958e-02,  1.75134733e-03],
        ...,
        [-9.41206294e-04,  6.76262099e-03, -2.90173246e-03, ...,
          6.98973471e-03,  1.07484020e-03, -9.42628551e-03],
        [ 1.01271353e-03, -8.72255955e-03, -1.04788346e-02, ...,
          2.06677560e-02, -7.23807840e-03, -2.96855788e-03],
        [ 4.51987609e-03, -9.64698754e-03, -1.28706265e-02, ...,
          5.09347580e-03,  6.54938747e-04,  6.30662590e-03]],

[[ 1.29155982e-02, -3.87949613e-03, -1.73111539e-02, ...,
          7.87006505e-03,  1.79969054e-03, -1.99378282e-03],
        [-5.64360945e-03,  6.23272732e-03, -1.77104597e-03, ...,
         -6.94065122e-03, -2.06026528e-03,  2.52026925e-03],
        [ 1.09358048e-02,  1.07644992e-02,  5.83288679e-03, ...,
         -3.34625063e-03,  5.10147365e-04, -4.78237215e-03],
        ...,
        [-4.59554838e-04, -4.96668136e-03,  3.04644182e-03, ...,
         -3.48758954e-03,  1.03788963e-02,  3.95182613e-03],
        [ 4.34665894e-03,  4.57208185e-03, -8.29102844e-03, ...,
          3.36311455e-03, -5.28571196e-03, -2.16717320e-03],
        [ 3.87410005e-03,  2.15103012e-03,  2.00013909e-02, ...,
         -2.00639721e-02, -5.61230956e-03, -1.40962899e-02]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.29023185, 0.30367318, 0.3277146 , ..., 0.29609588,
         0.28022346, 0.2690142 ],
        [0.31538793, 0.3181144 , 0.31960475, ..., 0.31267792,
         0.3197429 , 0.32831165],
        [0.27281317, 0.3185877 , 0.30232176, ..., 0.2977213 ,
         0.3166995 , 0.3241086 ],
        ...,
        [0.30449072, 0.29971388, 0.28448153, ..., 0.30133575,
         0.32245976, 0.29227644],
        [0.30001393, 0.29880044, 0.29624933, ..., 0.30133295,
         0.2868751 , 0.2922603 ],
        [0.34112212, 0.3260512 , 0.32140723, ..., 0.31743   ,
         0.3112761 , 0.29587394]]], dtype=float32)
                },
                'query': {
                    'b': array([[-0.00189125,  0.01352118,  0.00091355, ...,  0.0035183 ,
         0.00300467, -0.00666348],
       [-0.01087789,  0.02276341,  0.01180435, ...,  0.01987527,
         0.00734986, -0.03836636],
       [-0.00096058,  0.02366084,  0.03013458, ...,  0.02965958,
         0.03138456, -0.0436088 ],
       ...,
       [-0.00894505, -0.03361411,  0.03389145, ...,  0.03919084,
        -0.04814198,  0.0503398 ],
       [-0.02525179, -0.03671117, -0.0318046 , ...,  0.02517176,
         0.03593894, -0.01954985],
       [ 0.01204826, -0.02267273,  0.00109456, ..., -0.0036287 ,
         0.02977114, -0.040389  ]], dtype=float32),
                    'w': array([[[ 0.03751517, -0.00894794,  0.03796802, ...,  0.01834398,
          0.01501814, -0.01450621],
        [-0.01260894,  0.0070061 ,  0.01110138, ...,  0.013768  ,
         -0.02294332,  0.00360698],
        [ 0.00125167, -0.00402384,  0.00394119, ...,  0.00744796,
         -0.01517608,  0.00141339],
        ...,
        [-0.02915503,  0.03220129, -0.01296884, ..., -0.00667168,
         -0.00322573, -0.00568456],
        [ 0.00060752,  0.0221884 , -0.01554927, ...,  0.01631006,
          0.03953896, -0.02939055],
        [ 0.04179224, -0.02085465,  0.06698853, ..., -0.05321731,
          0.00048756, -0.06879911]],

[[ 0.00354561, -0.01446761, -0.0015612 , ...,  0.00956878,
         -0.00988728, -0.00204694],
        [ 0.02027416,  0.00979426,  0.00233873, ...,  0.01863737,
         -0.03342014, -0.02774412],
        [ 0.00776943, -0.00986928,  0.01352212, ...,  0.00450447,
         -0.01488789,  0.02115715],
        ...,
        [ 0.01297089, -0.00134582,  0.00226214, ..., -0.00276118,
         -0.00080995, -0.00480594],
        [ 0.00955805, -0.0036545 , -0.00687691, ..., -0.01106764,
          0.00188952,  0.00166115],
        [ 0.03332864, -0.00560288,  0.05338503, ..., -0.04126916,
          0.00984346, -0.04110722]],

[[ 0.00730518, -0.03303795,  0.02631902, ...,  0.01628805,
          0.00821777, -0.01016602],
        [ 0.01178334,  0.00690084,  0.03070587, ...,  0.01457074,
         -0.01117592, -0.0126626 ],
        [-0.00854391, -0.01213073,  0.00599931, ...,  0.00709854,
          0.01849573, -0.00779463],
        ...,
        [ 0.01797848, -0.0008457 , -0.01786283, ..., -0.00516583,
          0.01717598, -0.01582769],
        [ 0.01812719, -0.0155761 ,  0.00967004, ..., -0.00859368,
          0.00444214,  0.00816944],
        [ 0.01041352,  0.01317838,  0.05690333, ..., -0.05665426,
          0.02563898,  0.00091435]],

...,

[[-0.00784008,  0.0005012 ,  0.00181038, ...,  0.00581022,
         -0.01272136,  0.00411645],
        [-0.01476199,  0.0022948 ,  0.0016739 , ..., -0.00066222,
         -0.00838623,  0.00770193],
        [ 0.00392744,  0.02292484,  0.00547163, ...,  0.01098883,
          0.0046332 , -0.0036854 ],
        ...,
        [ 0.00223484, -0.01479245,  0.00490762, ...,  0.00892042,
          0.00948964, -0.00574697],
        [-0.00999111,  0.00116193,  0.00630799, ...,  0.01500735,
          0.0056859 ,  0.01637676],
        [-0.00336139, -0.01530121, -0.00614916, ...,  0.00717271,
          0.0028977 ,  0.0025799 ]],

[[-0.0159334 , -0.00037083,  0.00648348, ...,  0.01266751,
         -0.00111138,  0.02182904],
        [ 0.00131355,  0.00254501, -0.00177392, ...,  0.00338395,
          0.00468593,  0.01206343],
        [ 0.00476156,  0.0251711 ,  0.03167973, ...,  0.03343714,
          0.04333669, -0.02082606],
        ...,
        [ 0.01014087, -0.00841562,  0.00965948, ...,  0.01934951,
         -0.0088165 ,  0.01612883],
        [-0.04032471,  0.00143296, -0.00454972, ...,  0.00234946,
          0.0202724 , -0.01423668],
        [ 0.00861003, -0.01090591,  0.00929717, ..., -0.00694091,
          0.00800909, -0.02726219]],

[[ 0.01030037,  0.0047197 , -0.01638303, ..., -0.00018055,
          0.00406381, -0.00613182],
        [-0.00418899,  0.01524503,  0.01673017, ...,  0.01751116,
          0.0052884 , -0.03013111],
        [ 0.00471848,  0.00378889, -0.01872631, ..., -0.00934308,
          0.00240882,  0.00095373],
        ...,
        [ 0.00183086,  0.00629113, -0.00941781, ..., -0.01260495,
          0.01857397, -0.01199472],
        [-0.02759313, -0.00985234, -0.0257144 , ..., -0.00575151,
          0.01219597,  0.0021461 ],
        [ 0.0070406 ,  0.00050883, -0.00333748, ...,  0.000177  ,
         -0.00105772, -0.02623683]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.42830154, 0.41300058, 0.42222607, ..., 0.41873953,
         0.3830284 , 0.44085422],
        [0.5157273 , 0.5433036 , 0.45322174, ..., 0.46365637,
         0.51123625, 0.4334654 ],
        [0.4580163 , 0.4711203 , 0.38261098, ..., 0.40011474,
         0.5078213 , 0.4544825 ],
        ...,
        [0.40233824, 0.4881816 , 0.40679803, ..., 0.47694033,
         0.4520148 , 0.4807785 ],
        [0.5430778 , 0.43052778, 0.5226196 , ..., 0.48869303,
         0.6043722 , 0.6239678 ],
        [0.51000226, 0.60829395, 0.5999447 , ..., 0.55214936,
         0.42417204, 0.6633572 ]]], dtype=float32)
                },
                'value': {
                    'b': array([[ 0.00036897, -0.00305424, -0.00165733, ...,  0.00438358,
         0.00245094, -0.00461676],
       [-0.00301459, -0.00046264,  0.00285859, ...,  0.00075003,
        -0.00377895, -0.00295171],
       [ 0.00346862, -0.0006464 ,  0.00226621, ..., -0.00150375,
         0.00315236, -0.00238282],
       ...,
       [-0.01638476, -0.01227442,  0.01256817, ...,  0.0081502 ,
        -0.00806242, -0.01211064],
       [-0.00075014,  0.0007899 , -0.00100226, ...,  0.00551138,
         0.00104468,  0.00104668],
       [-0.00163955,  0.00094343, -0.00013186, ...,  0.00078494,
         0.00384166,  0.0019375 ]], dtype=float32),
                    'w': array([[[ 0.02118528,  0.02677314, -0.01200122, ...,  0.0262476 ,
          0.00594873, -0.02412984],
        [-0.00146269,  0.00968919, -0.03307377, ..., -0.00141116,
          0.01191948,  0.0207875 ],
        [-0.00940079, -0.00820884, -0.01442705, ...,  0.0038729 ,
         -0.03221527,  0.00978253],
        ...,
        [-0.0257416 , -0.01623189,  0.00299609, ...,  0.01148807,
         -0.00362875,  0.00621605],
        [-0.00116569, -0.02838015,  0.00101068, ...,  0.0427529 ,
          0.01070582, -0.0075488 ],
        [-0.00917997,  0.01766798,  0.00992273, ..., -0.00431697,
          0.02878406,  0.00446529]],

[[ 0.00702127,  0.00729654, -0.01880625, ...,  0.01480054,
          0.012403  , -0.00059675],
        [ 0.00408339,  0.00780023, -0.03327817, ..., -0.00935841,
          0.00450886,  0.02172546],
        [-0.00089169,  0.02752203, -0.00398778, ..., -0.01975279,
         -0.02512549, -0.02911799],
        ...,
        [-0.00399039,  0.00070716,  0.00131172, ..., -0.0002523 ,
         -0.00302091, -0.0075126 ],
        [-0.00937755, -0.01139442, -0.00019916, ...,  0.01939085,
          0.002853  ,  0.0200783 ],
        [-0.03255282, -0.00968802,  0.01254367, ..., -0.00310431,
          0.00760247,  0.01651519]],

[[-0.00287205,  0.0055108 ,  0.00055102, ...,  0.00216927,
          0.00150822, -0.00773935],
        [ 0.0129067 ,  0.0037404 , -0.00477334, ...,  0.01788685,
          0.00959588,  0.01789464],
        [ 0.00669868,  0.01335367,  0.00816026, ..., -0.01932909,
         -0.00625066, -0.00114229],
        ...,
        [ 0.00197083,  0.00637025,  0.00214494, ..., -0.01219716,
         -0.00771474,  0.00308848],
        [-0.01627306, -0.01433394,  0.0028006 , ...,  0.01064637,
         -0.00829327,  0.00128058],
        [-0.00290728, -0.00652776,  0.013403  , ..., -0.0025467 ,
          0.02734178, -0.00783805]],

...,

[[-0.00527864,  0.00207289, -0.0057024 , ...,  0.00735172,
         -0.0053231 ,  0.00311593],
        [-0.00387735,  0.00040894, -0.01830024, ..., -0.01362675,
          0.00399212, -0.00613649],
        [-0.00169294,  0.00483906, -0.00422814, ...,  0.0127602 ,
         -0.00649252, -0.00271095],
        ...,
        [ 0.00572385, -0.00170601,  0.00129312, ..., -0.0020116 ,
          0.00271703, -0.00179498],
        [ 0.00259946, -0.00730267,  0.0133883 , ..., -0.0077728 ,
          0.0012883 ,  0.00106838],
        [-0.00630673,  0.00132766, -0.00400298, ..., -0.00389056,
          0.00296963, -0.00973548]],

[[-0.00339014, -0.00682116,  0.01323037, ..., -0.00548842,
          0.01368415,  0.00751632],
        [-0.02050336,  0.00885444, -0.00676099, ...,  0.01518234,
          0.00349457,  0.00050448],
        [ 0.00357184,  0.0258451 ,  0.00325336, ...,  0.01683482,
         -0.00651918, -0.00410566],
        ...,
        [-0.01679386, -0.01334426,  0.00190475, ...,  0.00140239,
         -0.00553374, -0.02021114],
        [ 0.00601098, -0.0035773 ,  0.00361824, ..., -0.01299598,
          0.00575497,  0.00915871],
        [-0.01235994,  0.00768862,  0.01194046, ..., -0.01922636,
          0.01222123, -0.00598872]],

[[-0.02139034, -0.00867905,  0.0218287 , ..., -0.00269548,
         -0.01127221,  0.00096784],
        [ 0.00528475, -0.02709212,  0.00862739, ..., -0.01200321,
         -0.00751034,  0.01631586],
        [-0.01530002,  0.0075484 , -0.00682098, ..., -0.01271224,
         -0.00671244, -0.01073114],
        ...,
        [ 0.01245718, -0.00765948,  0.01388553, ..., -0.02158968,
          0.01284235, -0.00028408],
        [-0.00754987, -0.01706584, -0.00257025, ...,  0.00908255,
          0.0108436 ,  0.00761262],
        [ 0.00830173, -0.00970829, -0.00097407, ...,  0.02274511,
          0.01576199,  0.01823052]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.34201154, 0.32977283, 0.40947285, ..., 0.3541613 ,
         0.30881837, 0.28572962],
        [0.38379872, 0.39862862, 0.37658778, ..., 0.35451028,
         0.3734198 , 0.40331832],
        [0.31314212, 0.38785625, 0.34701166, ..., 0.3373546 ,
         0.3530489 , 0.4031688 ],
        ...,
        [0.3496849 , 0.33428174, 0.31916952, ..., 0.36040932,
         0.39962605, 0.35016263],
        [0.36046517, 0.3559316 , 0.33938995, ..., 0.36999813,
         0.2856677 , 0.34938678],
        [0.40142116, 0.40935415, 0.3983561 , ..., 0.31329104,
         0.35364568, 0.34154698]]], dtype=float32)
                }
            }
        },
        'x_layers_3': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': array([-0.00359166, -0.00079451, -0.00558264, ..., -0.00439165,
       -0.0099638 , -0.00504463], dtype=float32)
                    },
                    'linear': {
                        'w': array([[-0.06025301, -0.03328411,  0.03951382, ...,  0.03327454,
        -0.0440207 , -0.0191193 ],
       [ 0.01228818,  0.06975493,  0.01588618, ...,  0.02443731,
         0.00978233, -0.03454997],
       [-0.0288316 ,  0.00868899, -0.00439149, ..., -0.03457052,
        -0.01619829,  0.00575076],
       ...,
       [ 0.04839077,  0.01288755, -0.04040444, ...,  0.00701007,
        -0.01534467,  0.05396933],
       [ 0.02751558, -0.05551632,  0.01889827, ..., -0.01203401,
        -0.04030662, -0.02222754],
       [-0.03396595, -0.00715171,  0.02327061, ..., -0.02334552,
         0.03825033, -0.01363626]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.101636 , 1.5102721, 1.0762154, ..., 0.9510079, 1.0553099,
        1.1028898]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': array([ 0.00069713, -0.00100389, -0.0011128 , ...,  0.00128676,
        0.00130776, -0.00264015], dtype=float32)
                    },
                    'linear': {
                        'w': array([[ 0.04697682, -0.02177855,  0.0434398 , ...,  0.00020704,
         0.0306547 , -0.02656312],
       [ 0.05644013,  0.03143594, -0.03181333, ..., -0.00783562,
         0.06982915, -0.01424309],
       [ 0.01412346, -0.01519689,  0.02326811, ..., -0.00681298,
        -0.00771268,  0.01557296],
       ...,
       [ 0.04761029, -0.02560747, -0.0354092 , ..., -0.0507902 ,
         0.00030733,  0.00124969],
       [ 0.05086803,  0.00317969, -0.01772834, ..., -0.02629702,
         0.03743565, -0.03532964],
       [-0.03264811,  0.0267511 , -0.04038355, ..., -0.01169836,
        -0.030372  ,  0.0573629 ]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.0751002, 1.160724 , 1.084587 , ..., 1.0613495, 1.1038988,
        1.0690465]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': array([ 0.00047155,  0.00078917, -0.00076015, ..., -0.00538683,
       -0.00738913,  0.0017588 ], dtype=float32),
                    'scale': array([-0.01322251, -0.02040597, -0.01582962, ..., -0.00986545,
       -0.00596192,  0.00256199], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': array([1.0655224, 1.1028657, 1.013484 , ..., 1.0014967, 1.0687963,
       1.0570511], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': array([[-1.6785270e-03,  1.3397117e-03, -1.0215957e-03, ...,
         5.7368824e-04,  1.0545085e-03, -3.7307583e-03],
       [-1.0163713e-03, -9.7007185e-05,  6.2050847e-03, ...,
         2.6493550e-03,  2.1892705e-03, -2.9838490e-03],
       [-1.2241573e-03,  6.9353578e-04, -2.3339754e-03, ...,
        -3.0547809e-03, -6.9943204e-04,  1.5564183e-04],
       ...,
       [-6.0133091e-03,  6.9217617e-03,  6.3362829e-03, ...,
         2.1950218e-04, -1.9627574e-03, -5.8280532e-03],
       [-1.9570033e-03,  7.5407821e-04,  2.3321738e-04, ...,
        -3.1344227e-06, -1.0981299e-03, -4.7900848e-04],
       [-1.5198577e-03,  8.5386733e-04, -4.1893954e-04, ...,
         3.1281132e-04, -7.4202148e-04,  4.6884449e-04]], dtype=float32),
                    'w': array([[[ 6.0837939e-03,  4.6521290e-03,  7.5706211e-04, ...,
          9.1714406e-04, -3.6846860e-03, -5.4185386e-03],
        [-6.0345409e-03,  2.2672977e-02,  8.0101583e-03, ...,
         -5.0341138e-03, -2.5518978e-02,  1.1301177e-02],
        [ 1.6000951e-03,  1.6028784e-02,  4.7602216e-03, ...,
          4.7910311e-03,  1.6524794e-02, -2.1223988e-02],
        ...,
        [-3.4377207e-03,  3.5684025e-03,  8.2413079e-03, ...,
          9.2745209e-03,  1.6370324e-02, -9.4568986e-04],
        [ 1.3595778e-03,  1.3690447e-03,  2.4898786e-02, ...,
          5.3510466e-03, -2.4402350e-05, -1.7150324e-02],
        [-2.2716202e-02,  1.7809145e-02, -9.5531344e-03, ...,
          8.5292039e-03,  2.0799288e-03,  1.3175344e-02]],

[[ 7.7007082e-03, -1.6660610e-02,  1.8694381e-03, ...,
         -3.6377299e-03, -1.6561039e-02,  7.4376236e-03],
        [-1.7417083e-02,  7.2829132e-03, -1.7788141e-03, ...,
          3.1899894e-04, -8.4746527e-03, -1.5545859e-02],
        [ 2.5252276e-04, -1.4218090e-02,  4.6571018e-04, ...,
          1.0496932e-03,  3.1701021e-02, -6.3035204e-03],
        ...,
        [ 1.3730018e-02, -3.6516739e-03, -9.3931183e-03, ...,
          3.5420901e-03,  2.6293408e-02,  4.7906255e-04],
        [ 7.0330063e-03,  2.0069808e-02,  1.5945941e-02, ...,
         -6.3601448e-03,  9.1827632e-04, -2.7817989e-02],
        [-1.1937161e-03,  9.7999685e-03, -4.9198153e-03, ...,
          1.0826681e-02,  2.0052245e-02,  1.2130483e-02]],

[[-1.2842728e-02,  9.2336256e-03,  7.5104940e-03, ...,
          2.2071470e-02, -1.4607210e-02, -8.2422327e-03],
        [ 1.7439186e-03,  7.9496140e-03,  1.3051025e-02, ...,
          8.6997263e-03, -2.2245018e-02,  2.3979617e-04],
        [-2.0984549e-02, -6.0859839e-03, -1.1758319e-02, ...,
          2.4514729e-03, -2.9055625e-03,  8.4922416e-03],
        ...,
        [ 2.0849383e-02, -4.2151567e-03, -5.2765324e-03, ...,
          2.4988316e-02,  1.8651700e-02,  8.2285255e-03],
        [-1.3612515e-02, -2.8463767e-03,  1.6124411e-02, ...,
          2.4231630e-03,  2.3506312e-03, -7.9287104e-03],
        [-2.6979001e-02,  1.4218629e-03, -1.6566893e-02, ...,
         -1.0951935e-02,  2.9146362e-02,  1.6274336e-03]],

...,

[[-1.4308532e-02,  2.6006898e-02,  8.3279365e-04, ...,
          1.0420426e-02, -1.1879888e-02, -1.1360109e-02],
        [-7.6534795e-03, -8.6906804e-03, -1.3870924e-02, ...,
         -2.2638981e-04,  2.6050315e-03, -2.5221994e-03],
        [ 1.3970997e-02,  2.6322620e-03, -1.2131684e-02, ...,
          1.0120703e-02, -3.0151991e-02,  2.4127827e-03],
        ...,
        [-7.7244015e-03,  8.4552523e-03, -1.2768571e-04, ...,
          3.9877119e-03, -4.9953670e-03, -2.9930049e-03],
        [ 1.1555938e-03,  3.6788362e-03,  2.9020954e-03, ...,
         -1.4570947e-02, -6.8174801e-03,  5.8022551e-03],
        [-1.9016817e-02,  2.5822097e-02, -2.1856066e-02, ...,
         -2.1041792e-02,  2.4438476e-02, -2.1616530e-02]],

[[-3.4770777e-03, -2.7274514e-02, -4.5081954e-03, ...,
         -1.4774953e-02,  9.6199289e-03, -7.7483339e-05],
        [ 1.9727057e-02,  3.1924775e-04, -1.2746050e-02, ...,
         -1.2754171e-02,  1.1252610e-02,  8.5274577e-03],
        [-2.0253411e-02,  2.8057959e-02, -3.2061639e-03, ...,
         -2.9442557e-03, -5.6172600e-03,  2.0232843e-02],
        ...,
        [-1.4451443e-02,  1.7209789e-02,  6.9214115e-03, ...,
          6.3530942e-03, -4.1163675e-03, -6.1847856e-03],
        [-7.0819738e-03, -1.1343397e-02,  2.1781456e-02, ...,
          1.4147011e-02, -3.0891870e-03, -5.3311535e-03],
        [ 3.9609015e-02, -5.1715523e-03, -5.4990370e-03, ...,
         -3.0258808e-02,  1.0024987e-03,  2.7532536e-03]],

[[ 5.9171738e-03, -4.3242317e-04, -3.3961914e-02, ...,
         -2.6869291e-02, -1.5355121e-05,  7.3913564e-03],
        [-1.7731775e-03, -3.9709383e-03,  3.4796234e-04, ...,
         -1.1495024e-03,  4.7223768e-03,  1.2998351e-02],
        [-1.9058391e-02, -1.8019665e-02,  1.3039748e-02, ...,
          1.6119709e-03, -5.7866625e-03, -6.1591226e-03],
        ...,
        [-1.1477595e-02, -6.9808867e-04,  6.9915270e-03, ...,
         -1.6907848e-02,  2.3044579e-02, -8.4827431e-03],
        [-1.1313060e-02,  3.2085020e-04, -5.4306183e-05, ...,
         -6.9154515e-03, -1.3521667e-02,  1.3901893e-03],
        [ 1.0984845e-03,  1.9725360e-02, -1.0533265e-02, ...,
         -2.8858619e-04, -1.2176596e-02, -1.1769810e-03]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.40796515, 0.4098488 , 0.4332819 , ..., 0.38659358,
         0.48619506, 0.36847043],
        [0.40022975, 0.43262523, 0.36621147, ..., 0.3867601 ,
         0.40487224, 0.41619906],
        [0.5055723 , 0.54978985, 0.4410098 , ..., 0.49363375,
         0.4424238 , 0.6074698 ],
        ...,
        [0.33701968, 0.3650918 , 0.30900756, ..., 0.34762067,
         0.49402076, 0.31611723],
        [0.47811568, 0.37851104, 0.5255159 , ..., 0.39023367,
         0.43438235, 0.4930158 ],
        [0.55661035, 0.46944535, 0.45578927, ..., 0.5200823 ,
         0.5553281 , 0.47902992]]], dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': array([0.08037373, 0.07929973, 0.05286393, 0.12249207, 0.07311964,
       0.06011745, 0.06764143, 0.08867034, 0.07810722, 0.11073345,
       0.10193466, 0.05869003, 0.08364506, 0.07253328, 0.0851353 ,
       0.0989499 , 0.06423612, 0.06726725, 0.06934468, 0.06810744,
       0.08061647, 0.06345143, 0.06148439, 0.08881178, 0.09636571,
       0.07103957, 0.07336371, 0.09454832, 0.07582602, 0.12069155,
       0.07495704, 0.05750223, 0.09781589, 0.10714186, 0.06152625,
       0.11027586, 0.07866532, 0.13079058, 0.10691793, 0.13633348,
       0.0731928 , 0.05952192, 0.0733497 , 0.07639648, 0.05115701,
       0.0510252 , 0.09616829, 0.07447171, 0.05894472, 0.05770541,
       0.10170057, 0.05173   , 0.08556115, 0.08349261, 0.08772631,
       0.05666647, 0.05503659, 0.07228909, 0.08734103, 0.06599794,
       0.07190575, 0.06675142, 0.12766062, 0.07212538, 0.06819951,
       0.0679538 , 0.05753606, 0.11283694, 0.11276623, 0.07136308,
       0.06852216, 0.07107771, 0.07141783, 0.07693509, 0.09175616,
       0.05311742, 0.0721948 , 0.05768591, 0.08882511, 0.0579024 ],
      dtype=float32)
                },
                'post': {
                    'b': array([ 0.00054931, -0.00043755,  0.00115337, ...,  0.00313751,
        0.00021384, -0.00234471], dtype=float32),
                    'w': array([[[-1.54192438e-02,  1.00835366e-02, -5.50007075e-03, ...,
          1.30757838e-02,  3.85335903e-03,  8.46714713e-03],
        [ 1.19337235e-02, -7.40378397e-03, -8.98601767e-03, ...,
          1.74418994e-04,  9.26105119e-03,  1.36058349e-02],
        [-1.26508959e-02, -1.19769142e-03,  2.10273359e-02, ...,
         -1.45233786e-02, -1.35016337e-03,  1.32047795e-02],
        ...,
        [-2.17427174e-03,  3.66755878e-04, -1.46679357e-02, ...,
          1.28410952e-02,  1.63965933e-02,  1.18775107e-03],
        [-1.10554257e-02, -1.21107725e-02, -1.38095804e-02, ...,
         -4.13777167e-03,  2.90608988e-03,  1.77041776e-02],
        [ 6.82405941e-03,  2.09691795e-03,  4.36124858e-03, ...,
         -1.91892106e-02,  1.02002954e-03, -1.25868535e-02]],

[[ 2.47178902e-03, -9.83526697e-04,  1.91599340e-03, ...,
          1.98956160e-03, -2.39147041e-02,  5.76870330e-03],
        [ 2.36218031e-02, -2.76064547e-03,  1.36520471e-02, ...,
         -1.04848342e-02,  9.44505271e-04,  1.57460086e-02],
        [-4.10368433e-03, -7.51612289e-03,  3.03633208e-03, ...,
          2.40270817e-03, -8.15129373e-03, -2.54857959e-03],
        ...,
        [-7.30164163e-03,  8.08597601e-04, -1.57655273e-02, ...,
         -3.41438176e-03,  7.02309841e-03, -7.69510493e-03],
        [ 2.30727112e-03, -2.54571525e-04,  1.59401391e-02, ...,
          1.53160002e-02,  9.18446202e-03,  2.28550285e-03],
        [ 9.93945636e-03,  1.08048292e-02, -7.31418328e-03, ...,
          3.45523702e-03,  8.60742666e-03, -1.47478757e-02]],

[[ 1.11284899e-02, -7.48150889e-03, -1.07095810e-02, ...,
          3.12889065e-03, -7.11501949e-03, -7.67704053e-03],
        [ 3.70361991e-02,  6.01579715e-03,  9.04711802e-03, ...,
         -4.89746640e-03,  1.80390142e-02,  1.69262681e-02],
        [-1.98537335e-02, -1.82764009e-02,  2.12624343e-03, ...,
         -8.81485641e-03,  1.54320989e-02,  7.37081096e-03],
        ...,
        [ 7.89073389e-03, -4.46634414e-03, -1.55431079e-02, ...,
          6.51739072e-03,  5.70011465e-03, -4.90279775e-03],
        [-1.77413563e-03, -9.70605033e-05, -1.17600383e-02, ...,
          4.21141880e-03,  3.97652620e-03, -8.22904753e-04],
        [-1.19786814e-03,  1.54609950e-02,  1.64406013e-03, ...,
          1.30158793e-02, -8.10700376e-03, -1.44137209e-02]],

...,

[[-4.45628352e-03, -1.47522707e-02,  1.57147320e-03, ...,
         -3.44765885e-03,  1.00596715e-02, -6.25270943e-04],
        [-4.22204239e-03,  6.99051830e-04, -1.04098406e-03, ...,
          8.71939585e-03, -9.62650310e-03, -1.33748557e-02],
        [ 9.41813644e-03,  1.43713001e-02,  2.41950224e-03, ...,
         -4.32555936e-03, -5.54622756e-03,  4.68709972e-03],
        ...,
        [-1.13946274e-02, -4.04317491e-03,  2.75128288e-03, ...,
          1.04453694e-02, -4.21688752e-03, -6.76877610e-03],
        [ 1.77487098e-02,  2.30465028e-02,  1.12755187e-02, ...,
          1.28520811e-02,  1.57759164e-03, -5.63759077e-03],
        [-2.42867204e-03, -1.12820528e-02,  3.62545927e-03, ...,
          1.85468816e-03,  4.19222470e-03,  7.49515230e-03]],

[[-1.72932707e-02, -4.90112510e-03,  5.47851343e-03, ...,
         -1.32997185e-02,  1.13375057e-02,  2.56767264e-03],
        [-7.54152192e-03, -2.05422137e-02,  8.09672847e-03, ...,
         -5.39762899e-03,  1.12196654e-02, -7.56473094e-03],
        [ 2.15942394e-02,  1.79995913e-02, -2.72421557e-02, ...,
          1.55957099e-02, -2.04427447e-02,  1.50294136e-02],
        ...,
        [ 1.17960777e-02, -6.30891416e-03, -1.15472695e-03, ...,
          4.18155501e-03,  1.69985555e-02,  2.98294402e-03],
        [ 3.34308040e-03, -8.98825284e-03,  2.72085454e-04, ...,
         -1.11389707e-03,  1.94550099e-04,  5.30481339e-04],
        [ 8.79578572e-03, -2.31942302e-03, -9.34939831e-03, ...,
         -2.58195773e-03,  1.50033450e-02, -1.01400316e-02]],

[[ 2.22686562e-03,  2.18903972e-03,  1.50927959e-03, ...,
          1.07232034e-02, -1.32909827e-02,  1.15348352e-02],
        [ 1.62409302e-02, -2.02017254e-03, -1.52577711e-02, ...,
          3.22888186e-03, -2.56273290e-03, -7.44445203e-03],
        [-4.24485304e-04, -1.38442023e-02,  2.75478279e-03, ...,
          4.94189747e-03,  4.66359500e-03,  2.97783059e-03],
        ...,
        [-8.09659809e-03, -1.92497522e-02, -1.40441665e-02, ...,
          5.59709780e-03, -1.26869930e-03,  2.40132771e-03],
        [-8.65959469e-03,  1.91764440e-03, -1.21536665e-02, ...,
         -5.53380745e-03,  9.27343644e-05, -6.86470233e-03],
        [ 2.26667942e-03, -8.99902545e-03, -1.91209503e-02, ...,
         -7.43969483e-03, -2.70407391e-03,  1.19197704e-02]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.3441256 , 0.3397772 , 0.33512112, ..., 0.34742522,
         0.37539235, 0.3355507 ],
        [0.3671285 , 0.3167637 , 0.30823874, ..., 0.3351517 ,
         0.31705788, 0.3503268 ],
        [0.35279018, 0.36565453, 0.32973257, ..., 0.33539888,
         0.33381772, 0.3387922 ],
        ...,
        [0.32376596, 0.29141974, 0.30451894, ..., 0.30668443,
         0.30242306, 0.29465637],
        [0.38816893, 0.38387504, 0.40651184, ..., 0.4347907 ,
         0.35136265, 0.36932868],
        [0.32980263, 0.33508638, 0.35941473, ..., 0.37657708,
         0.34445637, 0.33982304]]], dtype=float32)
                },
                'query': {
                    'b': array([[-0.0063142 , -0.00455809,  0.02359539, ...,  0.03004604,
        -0.01363834, -0.01749385],
       [-0.00883258,  0.01348464, -0.01877279, ..., -0.00789092,
        -0.00137584,  0.00400858],
       [ 0.00920692,  0.00321167,  0.01175659, ...,  0.00476064,
        -0.00365732,  0.00989715],
       ...,
       [-0.02695556,  0.02235626,  0.01659154, ..., -0.02419706,
        -0.00562116, -0.02516452],
       [ 0.03666452,  0.03104526,  0.03207183, ..., -0.01275486,
        -0.00320612, -0.01301267],
       [ 0.00257295,  0.0046639 , -0.01591201, ..., -0.01375522,
         0.03187206, -0.00602493]], dtype=float32),
                    'w': array([[[ 1.65672582e-02,  3.58093791e-02,  1.30947670e-02, ...,
          1.43328891e-03,  4.70675714e-02,  2.24824739e-03],
        [-1.48226786e-02,  1.66079383e-02, -3.66222626e-03, ...,
          7.23349676e-03, -1.60129054e-03, -1.37269692e-02],
        [-1.83571000e-02,  3.74624468e-02,  7.09347241e-03, ...,
         -5.79345413e-02,  2.28687190e-03, -6.03809953e-03],
        ...,
        [ 4.69985092e-03,  6.11597765e-03,  8.14028841e-04, ...,
         -5.70318429e-03,  1.45086637e-02, -8.69368576e-03],
        [ 8.65209382e-03,  1.80686526e-02,  3.40784229e-02, ...,
         -6.34136191e-03, -4.05705571e-02,  2.96484702e-03],
        [-1.02002937e-02,  1.63281299e-02,  2.41871784e-03, ...,
          4.32097306e-03,  1.67190302e-02, -2.59064417e-02]],

[[-6.28292281e-03,  3.68267186e-02,  2.50358731e-02, ...,
          1.18682608e-02,  2.67039128e-02, -1.06200129e-02],
        [-8.68187635e-06,  2.72787753e-02,  1.13879619e-02, ...,
          2.54863054e-02,  1.88193489e-02, -1.50925834e-02],
        [-1.51366284e-02,  3.83543298e-02, -1.82455201e-02, ...,
         -3.60090733e-02,  9.94380191e-03, -3.52676511e-02],
        ...,
        [-5.91526879e-03, -3.24163958e-03,  1.26993824e-02, ...,
         -7.52043026e-03,  8.96463357e-03, -1.70528777e-02],
        [ 2.93393265e-02,  4.29840153e-03,  3.90925184e-02, ...,
          2.04838067e-02, -1.62950233e-02,  8.97464622e-03],
        [-2.03499966e-03, -8.84886133e-04,  9.59112402e-03, ...,
         -3.64136361e-02,  7.86823861e-04,  2.03779228e-02]],

[[-1.38067463e-02,  1.86806619e-02, -5.07931923e-03, ...,
          1.50973927e-02,  9.05765407e-03, -3.17620113e-02],
        [ 1.48867508e-02,  9.23980121e-03, -9.70555749e-03, ...,
         -5.10725379e-03, -1.77525487e-02,  6.70349039e-03],
        [-8.21147487e-03,  1.65300854e-02, -7.06808595e-03, ...,
         -1.75039601e-02,  1.41881891e-02, -2.16389839e-02],
        ...,
        [-1.89900976e-02,  1.79670844e-02,  8.12662579e-03, ...,
         -1.17659522e-02,  1.94190145e-02, -1.51595017e-02],
        [ 4.87595703e-03,  7.56143313e-03,  2.90022083e-02, ...,
          1.39921159e-02, -1.22405719e-02,  2.34070630e-03],
        [-7.11595826e-03,  1.91656630e-02, -1.08919004e-02, ...,
         -1.24339098e-02, -2.78134141e-02,  1.30391736e-02]],

...,

[[-2.41281488e-03,  8.62330198e-03, -1.53779089e-02, ...,
          1.08867008e-02, -7.51790404e-03, -1.03229973e-02],
        [-1.17751248e-02,  1.73449758e-02, -8.25022068e-03, ...,
          1.35458140e-02, -1.46147888e-02,  8.64077359e-03],
        [-1.58518236e-02, -9.80533659e-03,  3.30934743e-03, ...,
         -1.41412662e-02, -1.79620143e-02, -1.31845847e-02],
        ...,
        [-5.26336255e-03,  1.03837056e-02,  1.71405007e-03, ...,
          9.55697615e-03, -1.06510557e-02, -1.32496003e-02],
        [ 2.34516375e-02,  3.24509330e-02, -7.65971560e-03, ...,
         -1.32641904e-02, -2.32304912e-03, -3.46138850e-02],
        [-1.84765062e-03,  1.88855398e-02, -2.12441441e-02, ...,
          1.03242183e-02,  1.58255026e-02, -3.69849131e-02]],

[[-6.66872924e-03,  1.61805761e-03,  7.28358421e-03, ...,
          1.21308705e-02, -1.06915087e-02, -1.19230719e-02],
        [ 3.25712329e-03,  5.76933194e-03, -7.25680171e-03, ...,
         -4.39543165e-02, -1.47555871e-02, -5.11604268e-03],
        [ 5.25135722e-04,  5.39510977e-03,  5.49668726e-03, ...,
          8.34900606e-03,  2.95278663e-03,  1.39246248e-02],
        ...,
        [-1.66024137e-02,  1.59081630e-02,  1.77776832e-02, ...,
         -4.08153888e-03,  2.52008941e-02, -1.77548248e-02],
        [-9.26962867e-03, -3.84518382e-04,  2.18859874e-02, ...,
         -1.08323991e-02, -2.88616903e-02, -4.58440091e-03],
        [-3.27871963e-02, -1.33493291e-02, -1.55274440e-02, ...,
          6.65737642e-03, -9.13732219e-03,  2.65267342e-02]],

[[-5.17609809e-03,  2.61216294e-02, -7.79005094e-03, ...,
         -1.43469609e-02, -1.07553983e-02, -4.63652145e-03],
        [ 1.22595262e-02, -1.52036808e-02, -1.28153181e-02, ...,
          1.21101914e-02,  8.61967448e-03, -1.13262469e-02],
        [-1.66602302e-02, -1.14014233e-02, -1.76152457e-02, ...,
         -9.42711439e-03,  8.81423708e-03, -6.95279287e-03],
        ...,
        [-8.93063005e-03,  1.06965024e-02,  5.01304539e-03, ...,
         -5.93680236e-03, -1.38604939e-02, -1.03369784e-02],
        [ 1.32499551e-02,  9.15231463e-03,  7.31133949e-03, ...,
         -1.11583835e-02,  5.18290140e-03,  9.23759025e-03],
        [-5.59666334e-03, -6.71564310e-04,  1.70062017e-02, ...,
          3.26499939e-02,  2.08929162e-02, -8.61088000e-03]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.56957203, 0.5273579 , 0.5871176 , ..., 0.51412064,
         0.58269304, 0.5494279 ],
        [0.42711794, 0.40859804, 0.4741344 , ..., 0.49110052,
         0.4108554 , 0.46643543],
        [0.5951701 , 0.6253331 , 0.42668775, ..., 0.532124  ,
         0.5181197 , 0.6937366 ],
        ...,
        [0.44668788, 0.43793458, 0.3673043 , ..., 0.42632234,
         0.48339367, 0.39590484],
        [0.8264987 , 0.69542676, 0.69323975, ..., 0.6476746 ,
         0.5556129 , 0.5891921 ],
        [0.62246406, 0.65458083, 0.68600184, ..., 0.5570644 ,
         0.70982516, 0.6203339 ]]], dtype=float32)
                },
                'value': {
                    'b': array([[ 5.5404363e-04, -1.6793143e-03, -9.0165624e-05, ...,
        -6.5793184e-04, -1.2384902e-03,  9.3860185e-04],
       [ 2.4099937e-03, -6.8471888e-03,  6.3015427e-03, ...,
        -3.8942320e-03,  6.0042161e-03,  1.4533896e-03],
       [ 3.7580477e-03, -1.9677782e-03, -4.0561365e-04, ...,
         5.9572086e-03,  7.6286710e-04,  5.6843081e-04],
       ...,
       [ 2.3894291e-03,  2.4938518e-03, -2.3374441e-03, ...,
        -5.7953368e-03, -4.7948356e-03, -3.9126086e-03],
       [-2.7866389e-03, -2.4954984e-03, -1.5062568e-03, ...,
         3.5421588e-04, -5.6915311e-04,  1.1925710e-03],
       [ 2.6903339e-03,  2.7697408e-03,  1.0172953e-04, ...,
        -1.0963748e-04,  2.3099105e-03, -9.5514680e-04]], dtype=float32),
                    'w': array([[[-7.58182677e-03, -3.36851273e-03,  3.39671485e-02, ...,
          9.98453703e-03, -4.34027612e-03, -2.52746092e-03],
        [-1.20935272e-02,  2.24267375e-02,  7.97357119e-04, ...,
         -8.71001463e-03, -1.41596375e-02, -1.06950598e-02],
        [ 3.16537847e-03, -1.30682709e-02, -1.19612925e-02, ...,
         -7.02873431e-03,  3.03701207e-04,  5.01969084e-03],
        ...,
        [ 2.14373181e-03, -1.49115967e-02, -9.14988853e-03, ...,
         -3.12511139e-02,  7.74393207e-04, -9.39847156e-03],
        [ 2.83262059e-02, -1.54163834e-04,  7.94346852e-04, ...,
          2.94039142e-03, -2.18230039e-02, -2.59777851e-04],
        [ 1.26834419e-02,  2.08839588e-02,  1.99626083e-03, ...,
          1.73709262e-02, -1.05466777e-02, -2.94377445e-03]],

[[-2.78753764e-03, -1.45758251e-02,  1.83332786e-02, ...,
         -3.98861803e-03,  1.14400908e-02, -1.18512323e-03],
        [ 1.13537302e-03,  1.52344136e-02, -6.23505004e-03, ...,
         -2.02580006e-03, -8.33914522e-03, -1.38646113e-02],
        [ 1.28956977e-02,  4.86306362e-02,  8.68670701e-04, ...,
         -1.63255204e-02, -1.04658529e-02,  1.04594007e-02],
        ...,
        [ 2.06699613e-02, -6.96022669e-03, -1.87007934e-02, ...,
         -2.92465892e-02, -1.47229312e-02, -8.33052385e-04],
        [ 5.44113480e-03, -8.48115142e-03, -1.93515923e-02, ...,
          5.97232161e-03,  1.57099646e-02,  1.25472834e-02],
        [ 1.51766827e-02, -4.07750998e-03,  4.18459857e-03, ...,
          4.59868181e-03, -1.20603237e-02,  1.08099673e-02]],

[[-3.65603436e-03, -1.20756235e-02,  1.91282444e-02, ...,
          7.37649575e-03, -1.19519811e-02,  3.55226966e-03],
        [-1.72414463e-02,  7.43805198e-03, -3.78384429e-05, ...,
          6.98338076e-03, -8.05438403e-03, -3.45833949e-03],
        [ 1.02501558e-02,  8.87370296e-03,  1.25489989e-02, ...,
          7.13740243e-03,  5.21675451e-03, -5.99422399e-03],
        ...,
        [ 7.54501298e-03, -4.86122258e-03, -1.93756458e-03, ...,
         -1.73928458e-02,  2.81396974e-03,  6.61403080e-03],
        [ 4.87582292e-03, -7.55519699e-03,  1.47939576e-02, ...,
          8.55768006e-03,  1.06867927e-03,  6.21679146e-03],
        [-2.46133329e-03, -7.65146548e-03, -5.77244349e-03, ...,
         -5.05046919e-03, -3.30768619e-03,  4.07196628e-03]],

...,

[[ 7.66143529e-03,  5.52322110e-03, -7.45464861e-03, ...,
          2.47676438e-03,  7.64610479e-03, -5.28315140e-04],
        [ 8.83940514e-03,  2.70170812e-03,  6.05251594e-03, ...,
          2.55134539e-03,  4.11793066e-04,  2.99451232e-04],
        [ 1.02727842e-02,  1.70031132e-03, -1.38360932e-02, ...,
          5.43869473e-03, -7.13511230e-03,  3.59750981e-03],
        ...,
        [ 1.89254973e-02,  5.77297586e-04, -1.03384480e-02, ...,
         -1.28164683e-02,  6.78239018e-03, -9.66271199e-03],
        [-1.08416965e-02, -1.57571770e-02, -6.36279443e-03, ...,
         -2.71256152e-03,  7.58514507e-03,  1.23419260e-04],
        [ 3.72610544e-03,  3.55584826e-03, -8.48440640e-03, ...,
         -1.69135276e-02,  3.94226838e-04, -1.01137860e-02]],

[[-3.26345977e-03,  1.26496265e-02,  1.39645571e-02, ...,
         -7.61854881e-03,  3.07666766e-03,  1.37850242e-02],
        [ 9.80694126e-03, -1.36164553e-03,  6.89540757e-03, ...,
         -9.44174360e-03,  1.29302787e-02,  6.41821325e-03],
        [ 6.77134376e-04,  9.09451919e-04,  2.87717730e-02, ...,
         -1.09102996e-02,  8.70500971e-03,  1.26091307e-02],
        ...,
        [ 4.63000288e-05,  1.81355737e-02, -3.76242469e-03, ...,
         -1.24914171e-02,  6.46690279e-03, -1.01927193e-02],
        [ 8.95201229e-03, -2.87314411e-03,  2.09922418e-02, ...,
         -7.99709838e-03,  1.07341819e-02, -2.71255057e-03],
        [-8.22139531e-03,  3.33281420e-03,  2.96300929e-02, ...,
          2.18745023e-02, -1.40872020e-02,  5.88064606e-04]],

[[ 2.23035496e-02, -6.43958477e-03,  1.02769854e-02, ...,
         -5.03790658e-03,  1.96808223e-02, -3.42780426e-02],
        [-1.35366600e-02,  4.95954510e-03,  1.61573887e-02, ...,
          9.64686275e-03,  2.81135610e-04, -7.91399926e-03],
        [-2.77084392e-03,  5.36162453e-03,  1.09172724e-02, ...,
         -4.54607280e-03, -1.73717504e-03, -6.74237777e-03],
        ...,
        [ 3.01251691e-02,  1.58525016e-02,  1.94437802e-02, ...,
          1.11673763e-02,  9.36414767e-03, -2.37105903e-03],
        [ 1.60713494e-02, -5.39772771e-03,  1.84250027e-02, ...,
         -1.80582572e-02, -5.87218814e-03,  2.32455432e-02],
        [-4.67143068e-03, -9.34725441e-03,  5.61833940e-03, ...,
          5.75147616e-03,  1.40718175e-02,  2.38939351e-03]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.4142319 , 0.3949837 , 0.4075967 , ..., 0.4186205 ,
         0.45753047, 0.38567942],
        [0.44150716, 0.3541366 , 0.3573654 , ..., 0.402663  ,
         0.38835308, 0.44105473],
        [0.40475622, 0.44546866, 0.40716568, ..., 0.40128946,
         0.40113223, 0.40821117],
        ...,
        [0.4054993 , 0.35667482, 0.37096214, ..., 0.38152018,
         0.37164745, 0.35476995],
        [0.48808613, 0.44064978, 0.49935833, ..., 0.5165212 ,
         0.42456046, 0.43072966],
        [0.37430227, 0.38100886, 0.44260064, ..., 0.47486955,
         0.41577065, 0.41933876]]], dtype=float32)
                }
            }
        },
        'x_layers_4': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': array([-0.00033918, -0.00515734, -0.00025558, ...,  0.00321242,
       -0.0045813 ,  0.00603206], dtype=float32)
                    },
                    'linear': {
                        'w': array([[ 0.00432622, -0.03287012,  0.01275862, ...,  0.03733737,
        -0.00666521, -0.01324154],
       [-0.03115523,  0.03335258,  0.01877301, ...,  0.03141083,
        -0.00394512, -0.05304177],
       [ 0.00881899, -0.05999147,  0.00512024, ...,  0.01997933,
         0.04666889, -0.02512316],
       ...,
       [ 0.01407511, -0.01288593, -0.01321636, ...,  0.04786241,
         0.00889695,  0.03299772],
       [ 0.00216974,  0.01110496,  0.02895355, ..., -0.04200781,
         0.02919233,  0.03616085],
       [ 0.02409721, -0.03261668,  0.03869701, ..., -0.01949932,
         0.00740029, -0.02987003]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.0330187, 1.0914618, 1.1513411, ..., 1.0885044, 1.0164673,
        1.0796703]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': array([ 1.2490508e-03,  4.1453599e-04, -2.8177700e-03, ...,
        1.4040343e-04,  3.5180324e-03, -7.5857452e-06], dtype=float32)
                    },
                    'linear': {
                        'w': array([[-0.00270853,  0.007068  ,  0.00870072, ..., -0.04568937,
        -0.00149104, -0.04165388],
       [ 0.05179108, -0.02407867, -0.01176148, ...,  0.03194524,
        -0.02103354, -0.0012801 ],
       [ 0.01410516,  0.03630389,  0.00128203, ...,  0.03379128,
        -0.02607304,  0.04072599],
       ...,
       [ 0.0087957 , -0.01915103, -0.00403893, ...,  0.02046533,
         0.0604092 ,  0.05353087],
       [-0.03155645,  0.04317928,  0.02834019, ..., -0.03319148,
        -0.04890137, -0.02391277],
       [-0.03234796,  0.04266334,  0.01308641, ..., -0.00362083,
        -0.02230242,  0.00610088]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.103109 , 1.177369 , 1.1127108, ..., 1.1364425, 1.1226817,
        1.1492555]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': array([-0.0006323 , -0.00250285,  0.00583033, ...,  0.00232698,
        0.00085398, -0.00280266], dtype=float32),
                    'scale': array([-0.01994971,  0.00063523,  0.01054509, ..., -0.01481609,
        0.00990174, -0.01184579], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': array([1.1022189, 1.11532  , 1.0147704, ..., 1.016388 , 1.0738846,
       1.0579767], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': array([[-1.8876005e-03, -4.2480972e-04,  3.3034892e-03, ...,
        -6.5590523e-04, -5.9691512e-05,  1.5365365e-03],
       [-1.7957551e-03, -3.6221617e-03,  1.1712223e-03, ...,
        -6.4384378e-04,  1.7770033e-03,  5.6157954e-04],
       [-3.7401831e-03,  9.2523871e-04,  2.3624795e-03, ...,
         1.7385277e-03, -7.7581783e-05,  7.2749582e-04],
       ...,
       [-5.9823133e-03, -1.1551752e-03,  1.7608545e-03, ...,
         2.5742259e-03, -3.2627333e-03, -4.1508870e-03],
       [ 5.9462013e-04, -3.0149426e-05,  2.7025258e-04, ...,
         1.5758632e-03,  1.9301582e-03,  4.0970929e-04],
       [ 8.8718737e-04, -1.5026670e-03,  1.1991300e-03, ...,
        -2.1778243e-03, -1.1430072e-03, -9.6649473e-04]], dtype=float32),
                    'w': array([[[ 3.01337000e-02,  1.24209067e-02, -5.86472964e-03, ...,
         -8.65774229e-03, -1.08871749e-02,  2.49727443e-02],
        [ 2.31124274e-02, -1.10988012e-02,  1.84235850e-03, ...,
         -1.88146941e-02,  3.76610714e-03,  2.84295399e-02],
        [ 3.14491009e-03, -8.37674830e-03, -1.24650598e-02, ...,
         -6.69335388e-03, -4.62858845e-03,  5.41267218e-03],
        ...,
        [ 7.64358323e-03, -8.19855649e-03,  9.58123524e-03, ...,
         -6.77058846e-03, -1.36759533e-02,  6.71977177e-03],
        [ 4.26648557e-03, -9.02947877e-03, -8.26200470e-03, ...,
         -1.04721291e-02,  3.18084680e-03,  5.80366934e-03],
        [-2.57805432e-03,  5.40386792e-03, -1.25376377e-02, ...,
         -1.93149422e-03, -9.40163434e-03,  1.43176261e-02]],

[[-1.01587046e-02,  5.35686675e-04, -4.68355743e-03, ...,
         -1.10975746e-02,  2.91718636e-03, -8.39970727e-03],
        [-4.02158796e-04,  3.71387112e-03, -6.52795844e-03, ...,
         -8.34567659e-03,  1.14528555e-02, -3.19356844e-03],
        [ 6.76072715e-03, -4.67817625e-03,  1.06005259e-02, ...,
         -2.89717223e-02,  6.22447580e-03, -1.03421055e-03],
        ...,
        [-1.46741485e-02, -1.43816369e-02,  7.06730923e-03, ...,
         -4.48654057e-04, -1.84486788e-02, -2.30598282e-02],
        [ 1.34034315e-03, -7.61708664e-03,  4.11785347e-03, ...,
          2.61079762e-02, -2.55433656e-03,  1.93115212e-02],
        [-3.51030007e-03, -8.33391957e-03,  1.05529204e-02, ...,
         -2.18608715e-02,  6.33778400e-04, -9.15159285e-03]],

[[ 1.07570002e-02, -8.89669824e-03,  4.41863062e-03, ...,
          1.20124000e-03, -2.12726486e-03, -5.00235613e-03],
        [-1.48849944e-02, -2.56073568e-03, -9.82255582e-03, ...,
         -5.86179877e-03,  2.85189580e-02, -1.01815974e-02],
        [ 5.45540359e-03, -4.93233139e-03, -1.45831052e-02, ...,
         -2.87135784e-02,  2.29986757e-02, -5.70630841e-02],
        ...,
        [-9.14685335e-03, -1.13015119e-02,  7.98442215e-03, ...,
          3.14814551e-03, -2.01340877e-02, -5.15011372e-03],
        [-5.53619955e-03,  3.46078014e-05, -5.85266808e-03, ...,
          1.72657967e-02,  3.04380711e-03,  2.11280789e-02],
        [-1.26281623e-02, -9.51008499e-03,  3.21315490e-02, ...,
         -2.25253701e-02, -2.15680972e-02,  2.32509035e-03]],

...,

[[-1.04255062e-02,  1.48928631e-02, -1.73066985e-02, ...,
         -1.27893556e-02, -1.22909741e-02,  2.32807826e-03],
        [-7.39737554e-03,  3.24710901e-03, -1.51883950e-03, ...,
         -1.51323434e-03,  3.40778823e-03, -1.56587120e-02],
        [-1.19881593e-02, -6.74739189e-04, -9.68571258e-05, ...,
         -1.76143693e-03,  1.31908031e-02,  1.19968802e-02],
        ...,
        [ 5.06830448e-03,  6.66629989e-04, -8.15594010e-03, ...,
         -3.18155531e-03,  1.21726152e-02,  1.51970917e-02],
        [-9.90778301e-03, -1.11902775e-02,  1.35711348e-02, ...,
         -1.42871710e-02,  1.69108380e-02,  7.95321260e-03],
        [ 1.95181952e-03, -1.21148014e-02, -2.62625254e-05, ...,
          2.64731632e-03,  8.13155062e-03,  3.99310002e-03]],

[[ 9.60985199e-03,  8.81934445e-03,  1.57736726e-02, ...,
         -9.52556450e-03,  5.90825453e-04,  6.45682262e-03],
        [ 8.45667720e-03, -5.08341705e-04,  4.46949853e-03, ...,
         -6.96695363e-03, -9.12587531e-03,  1.38077801e-02],
        [ 8.55430122e-03,  6.51965616e-04,  6.42338023e-03, ...,
          5.79007156e-03,  6.12254208e-03,  2.38998160e-02],
        ...,
        [ 9.61323211e-04,  1.06490068e-02, -1.77535862e-02, ...,
          2.26073563e-02,  9.31188557e-03, -8.19957466e-04],
        [-8.65185540e-03,  2.11409517e-02, -1.55314077e-02, ...,
         -2.40438171e-02,  6.83145784e-03,  6.93235686e-03],
        [-1.79241747e-02,  1.92434546e-02, -2.08954923e-02, ...,
          3.81810404e-03,  1.81255713e-02,  2.23735422e-02]],

[[ 8.64202995e-03, -1.30942725e-02, -1.28558045e-02, ...,
         -1.35812210e-02,  1.13754417e-04,  1.10555319e-02],
        [-4.71161539e-03,  1.73984803e-02, -1.53125841e-02, ...,
          8.52911081e-03,  8.62509571e-03,  1.41716450e-02],
        [ 5.33405552e-03,  6.14917604e-03, -9.92153399e-03, ...,
          1.73857119e-02, -2.15481967e-02,  5.46277035e-03],
        ...,
        [-2.68412335e-03,  2.79641099e-04, -1.33013939e-02, ...,
          4.02520550e-03, -3.90526606e-03, -5.58624044e-03],
        [ 1.42477348e-03, -3.74906085e-04, -1.70411021e-02, ...,
          8.66159983e-03,  1.14939520e-02,  6.46173663e-04],
        [ 7.69440318e-04,  9.34678968e-03, -1.60742626e-02, ...,
          1.88496392e-02, -6.27175719e-03,  1.66990552e-02]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.4780104 , 0.5473833 , 0.47068864, ..., 0.4066451 ,
         0.4953522 , 0.40304443],
        [0.44623718, 0.3674018 , 0.38837376, ..., 0.45081985,
         0.43769404, 0.51594096],
        [0.5286036 , 0.41964746, 0.51784027, ..., 0.4401857 ,
         0.43516523, 0.5810773 ],
        ...,
        [0.34481898, 0.32435194, 0.37798542, ..., 0.37607336,
         0.32490626, 0.37671068],
        [0.39882225, 0.45797423, 0.4795707 , ..., 0.5078298 ,
         0.45115924, 0.4114221 ],
        [0.46900192, 0.5693813 , 0.5124517 , ..., 0.42943665,
         0.44571066, 0.48463976]]], dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': array([0.06925689, 0.0687267 , 0.09816898, 0.05614004, 0.10127976,
       0.09549069, 0.05992376, 0.1010856 , 0.07627991, 0.06381155,
       0.06614725, 0.11821901, 0.1060421 , 0.09320906, 0.08363871,
       0.09773096, 0.08945961, 0.08148175, 0.08691528, 0.05950036,
       0.1054969 , 0.07629689, 0.06731045, 0.05241217, 0.07037603,
       0.08807229, 0.08101945, 0.11336018, 0.08727027, 0.08233001,
       0.1025634 , 0.10394803, 0.10563974, 0.06789199, 0.15156211,
       0.0974873 , 0.09855369, 0.09362685, 0.08743744, 0.04419754,
       0.09561277, 0.08159087, 0.05990618, 0.12188151, 0.07485054,
       0.1204315 , 0.08618229, 0.09011715, 0.0703643 , 0.11945228,
       0.07089204, 0.10021815, 0.09990682, 0.08731607, 0.08622605,
       0.12677069, 0.10883871, 0.09831861, 0.09507571, 0.06917293,
       0.08881298, 0.10157361, 0.10612229, 0.10519458, 0.12216848,
       0.09660444, 0.10267104, 0.08772461, 0.0829059 , 0.11468416,
       0.08165301, 0.0679692 , 0.07408445, 0.07055859, 0.10889028,
       0.07661044, 0.08274958, 0.09309391, 0.06840506, 0.06962132],
      dtype=float32)
                },
                'post': {
                    'b': array([ 0.00018584,  0.00054333,  0.00215429, ...,  0.00092777,
        0.00437075, -0.00032677], dtype=float32),
                    'w': array([[[-0.00364268, -0.00533502,  0.02635869, ...,  0.00606002,
         -0.00487637, -0.00248202],
        [ 0.01556431,  0.02563867,  0.00874958, ...,  0.01622818,
          0.00538342, -0.00068508],
        [-0.00566908, -0.00174261, -0.00176214, ...,  0.00396739,
          0.01358565, -0.00684316],
        ...,
        [ 0.0211008 ,  0.001731  , -0.01537344, ..., -0.00073908,
          0.01017119, -0.01133754],
        [ 0.00807352, -0.02239311,  0.00981964, ..., -0.02785161,
          0.00785703, -0.0189785 ],
        [-0.0037584 ,  0.00413363, -0.0038328 , ...,  0.010292  ,
          0.00581   , -0.00778716]],

[[-0.00342621,  0.01195646,  0.00626059, ...,  0.01152808,
          0.01026207, -0.00670375],
        [-0.00334852, -0.01045291, -0.0139829 , ..., -0.00220572,
          0.01748258,  0.00353179],
        [ 0.00978813, -0.00473176,  0.00466992, ..., -0.016842  ,
         -0.00090071,  0.00342297],
        ...,
        [ 0.01260183,  0.00240335, -0.02869877, ..., -0.00041997,
          0.00289371, -0.0041544 ],
        [ 0.00620279,  0.0017847 , -0.00745507, ...,  0.01326036,
         -0.00269652, -0.00995864],
        [-0.02918411, -0.00758211, -0.02975841, ..., -0.00376696,
         -0.00125189, -0.00700308]],

[[ 0.008334  , -0.01058329,  0.015751  , ...,  0.008536  ,
         -0.00044655, -0.0222936 ],
        [-0.00575933, -0.01248068, -0.00184789, ..., -0.00101967,
          0.00494366, -0.0047694 ],
        [-0.01278478,  0.00728068, -0.01408311, ...,  0.00169644,
          0.01031604,  0.01785856],
        ...,
        [ 0.01845922, -0.00938113,  0.00632038, ..., -0.0038095 ,
          0.00217786,  0.00081036],
        [ 0.01022046,  0.012982  ,  0.00105982, ..., -0.00039202,
          0.00066632,  0.00583678],
        [-0.00244197,  0.00637915, -0.00827737, ...,  0.00834087,
          0.00061116, -0.0248445 ]],

...,

[[ 0.0030369 ,  0.00966552, -0.00393221, ..., -0.005367  ,
          0.00667013,  0.01778866],
        [ 0.01450602,  0.01011863,  0.00948474, ...,  0.00210128,
         -0.00989367, -0.00584687],
        [ 0.01079928,  0.00122925,  0.00896634, ...,  0.00517835,
         -0.00410692,  0.00940131],
        ...,
        [-0.00077921, -0.00052999, -0.00543942, ..., -0.00408598,
         -0.00378187,  0.00058528],
        [-0.00392729,  0.00071836, -0.00391443, ...,  0.00564476,
         -0.00647169, -0.00013269],
        [-0.01134015, -0.00937242,  0.0059218 , ...,  0.005702  ,
         -0.01232274,  0.00970433]],

[[ 0.0057896 , -0.00368114, -0.0056546 , ..., -0.00156527,
          0.00024567, -0.0058857 ],
        [ 0.01186835,  0.01077314,  0.0035416 , ...,  0.00102596,
         -0.00377653, -0.01403272],
        [-0.02751501,  0.00546371, -0.01826483, ...,  0.00295487,
         -0.00232483, -0.00175722],
        ...,
        [ 0.00012973, -0.00072872,  0.01904731, ...,  0.00320774,
          0.00959184,  0.00924393],
        [ 0.01500166,  0.02014527, -0.01031888, ...,  0.00988461,
         -0.00473079,  0.01564728],
        [-0.00077002, -0.0026033 ,  0.00387022, ..., -0.00716743,
          0.00121493,  0.00605872]],

[[-0.00093579,  0.00218967,  0.00322592, ..., -0.00076474,
          0.01170467, -0.00524056],
        [ 0.01144154, -0.00816934,  0.01511669, ..., -0.01021524,
         -0.01007736,  0.00074415],
        [-0.00469364, -0.00903361,  0.02324042, ..., -0.01450897,
          0.00737509, -0.00731732],
        ...,
        [-0.00158206, -0.00973321, -0.01214427, ..., -0.00271589,
         -0.00512956,  0.01256492],
        [-0.00496091, -0.00923559,  0.00735262, ...,  0.00423574,
         -0.01897817,  0.0084137 ],
        [-0.01685073,  0.02206317, -0.00042697, ..., -0.01593289,
         -0.0175844 , -0.01029996]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.35491452, 0.3506639 , 0.32426903, ..., 0.3552347 ,
         0.33343944, 0.33933896],
        [0.3304826 , 0.31611446, 0.35941574, ..., 0.31336945,
         0.35120383, 0.3141392 ],
        [0.3950579 , 0.3564111 , 0.395482  , ..., 0.38268226,
         0.37366176, 0.41941392],
        ...,
        [0.32176045, 0.3579692 , 0.35488227, ..., 0.31422317,
         0.3357452 , 0.31748614],
        [0.34581465, 0.34151173, 0.32222676, ..., 0.32698026,
         0.34721786, 0.33799317],
        [0.37102103, 0.3736726 , 0.3650617 , ..., 0.3352397 ,
         0.3410245 , 0.34606138]]], dtype=float32)
                },
                'query': {
                    'b': array([[-0.00904353,  0.00922141,  0.00253093, ...,  0.0071486 ,
         0.01894851, -0.00297219],
       [ 0.02580181, -0.01197189,  0.03771947, ..., -0.01448922,
         0.03808154,  0.0124295 ],
       [-0.00649451, -0.00354858,  0.00604098, ..., -0.00584646,
         0.00274818,  0.00644646],
       ...,
       [ 0.01927792,  0.00616192, -0.01234802, ..., -0.01084663,
         0.02252456,  0.01259941],
       [ 0.01681058, -0.01131948,  0.00902725, ..., -0.00339165,
        -0.0137673 , -0.00382652],
       [ 0.00305724,  0.00126192,  0.00328541, ..., -0.00471952,
         0.0095676 , -0.00190623]], dtype=float32),
                    'w': array([[[ 2.46237386e-02,  4.97382774e-04,  1.69206820e-02, ...,
          2.41074469e-02, -1.38304487e-03,  2.32416531e-03],
        [ 2.64098775e-02, -1.28170568e-02,  4.24231729e-03, ...,
         -1.04854880e-02,  1.35888867e-02,  2.55975388e-02],
        [ 3.81973125e-02, -1.20887049e-02, -4.18927427e-03, ...,
         -3.14308107e-02, -4.72382940e-02, -1.75638311e-02],
        ...,
        [ 2.11419575e-02,  6.90283673e-03, -3.24399546e-02, ...,
         -3.98736959e-03,  1.54951680e-02,  8.00326746e-03],
        [ 3.67254554e-03,  6.81026978e-03, -2.48883180e-02, ...,
         -8.82120710e-03,  2.65567023e-02,  6.48691785e-03],
        [-8.71083350e-04,  1.85516886e-02, -2.39707213e-02, ...,
         -1.15928007e-02,  1.32977040e-02,  1.84124187e-02]],

[[-1.00589460e-02, -2.27418412e-02, -4.12014173e-03, ...,
          1.02789300e-02,  2.46179048e-02, -2.12133247e-02],
        [ 8.32288489e-02, -4.68929708e-02,  2.44671609e-02, ...,
          1.81003064e-02, -2.95658838e-02,  2.57954672e-02],
        [ 4.38491218e-02, -1.71888378e-02, -6.43858984e-02, ...,
          7.04233116e-03, -4.20305356e-02, -1.80346351e-02],
        ...,
        [ 5.09687513e-03, -5.71056362e-03,  4.96693188e-03, ...,
         -6.15530089e-03, -1.38243325e-02,  7.48314336e-03],
        [-5.55190258e-03, -5.96179580e-03, -8.66231043e-03, ...,
          2.37964317e-02, -2.36401893e-03, -4.28230083e-03],
        [ 3.21977064e-02,  5.38986828e-03, -2.58978009e-02, ...,
         -2.21610479e-02,  3.00993416e-02,  2.17019697e-03]],

[[ 1.45136407e-02,  1.40780890e-02,  1.13277081e-02, ...,
          3.07448930e-03, -6.52937684e-03,  2.13428717e-02],
        [ 3.04943100e-02, -2.90186815e-02,  3.25217359e-02, ...,
          1.32216411e-02, -1.41715026e-03,  2.02751067e-03],
        [ 1.49908084e-02,  2.87255347e-02,  5.35836397e-03, ...,
          1.29024368e-02, -2.99836155e-02,  1.16677245e-03],
        ...,
        [ 1.06735080e-02, -6.98545575e-03,  5.04721887e-03, ...,
          1.49280496e-03, -7.52828084e-03,  8.74875765e-03],
        [ 6.42584683e-03, -2.52559893e-02, -3.15825939e-02, ...,
         -7.37268711e-03,  8.72115325e-03,  1.44710680e-02],
        [-1.09243235e-02, -9.48734023e-03, -2.84866281e-02, ...,
          5.93522890e-03,  2.00808682e-02,  4.84604016e-03]],

...,

[[ 5.37342345e-03,  2.10412666e-02, -1.77885629e-02, ...,
         -1.20420316e-02, -2.34429445e-03,  1.47947492e-02],
        [-7.19816843e-03, -3.51369232e-02,  3.07056606e-02, ...,
         -5.60689019e-03,  1.29475463e-02, -1.31174149e-02],
        [-2.03831866e-02,  7.41668558e-03,  1.59152988e-02, ...,
          5.11537492e-03,  1.44876149e-02, -1.63588603e-03],
        ...,
        [ 1.03449414e-03,  2.77529471e-03, -2.70526065e-03, ...,
          2.18044408e-02,  3.40628205e-04,  4.63171210e-03],
        [-2.83995178e-03, -1.91112757e-02,  5.78876911e-03, ...,
          6.25903122e-05, -2.81480839e-03,  2.28202194e-02],
        [-7.94511288e-03,  4.58106137e-04,  2.03445591e-02, ...,
          1.37764844e-03, -2.21148483e-03, -1.23551255e-03]],

[[-2.82299127e-02,  1.18036773e-02,  4.58430015e-02, ...,
          3.73046584e-02,  4.02544485e-03,  1.29068689e-02],
        [ 2.38386225e-02,  2.33831536e-03,  2.94055399e-02, ...,
          2.15208475e-02, -8.85839472e-05,  4.92629921e-03],
        [ 3.60063240e-02,  3.14452536e-02,  4.36873659e-02, ...,
         -2.68155523e-03, -3.37699503e-02,  1.99175738e-02],
        ...,
        [ 1.69939101e-02,  5.61214576e-04,  8.46868847e-03, ...,
         -2.15138886e-02, -7.11588515e-03,  7.73419347e-03],
        [ 1.45142491e-03,  1.22080008e-02, -4.20662016e-03, ...,
          8.36226624e-03,  2.09868830e-02, -3.91181698e-03],
        [ 3.98498103e-02, -3.93578317e-03,  4.62730378e-02, ...,
         -4.14667232e-03,  2.95646675e-03, -5.99747698e-04]],

[[-2.82171890e-02, -2.35241577e-02, -8.98296759e-03, ...,
         -2.17941846e-03,  3.43297906e-02, -2.92673726e-02],
        [-9.73666925e-03,  1.44522614e-03,  2.03186627e-02, ...,
          7.75819179e-03,  6.99952804e-03,  1.75368171e-02],
        [-1.80099886e-02,  1.42530501e-02,  2.05159020e-02, ...,
         -3.35845947e-02,  9.71715711e-03, -2.32642633e-03],
        ...,
        [ 1.70393952e-03,  3.93192488e-04,  1.15964804e-02, ...,
         -4.94584255e-03,  5.23317885e-03,  3.36621935e-03],
        [-2.14738287e-02,  2.95266155e-02,  2.52535124e-03, ...,
         -3.53639126e-02,  1.94096528e-02, -1.27472002e-02],
        [ 8.97306192e-04, -1.37348566e-02,  3.49913090e-02, ...,
         -8.99178721e-03,  5.32816956e-03, -1.90543346e-02]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.5830474 , 0.53612065, 0.70579624, ..., 0.66503197,
         0.5249813 , 0.5352495 ],
        [0.63160956, 0.5070762 , 0.61367106, ..., 0.55399   ,
         0.47870058, 0.55616814],
        [0.774811  , 0.6321415 , 0.70362437, ..., 0.7078728 ,
         0.6686057 , 0.6647644 ],
        ...,
        [0.42417032, 0.39790717, 0.4772963 , ..., 0.40933606,
         0.3291991 , 0.46010977],
        [0.5191574 , 0.6217304 , 0.62188417, ..., 0.6029309 ,
         0.54605633, 0.46816692],
        [0.6911394 , 0.5633411 , 0.6487666 , ..., 0.5861203 ,
         0.6968133 , 0.50759125]]], dtype=float32)
                },
                'value': {
                    'b': array([[ 2.1524420e-03,  2.9411949e-03,  1.0325930e-03, ...,
         3.1009023e-03, -1.1407599e-03, -1.3143895e-04],
       [-1.3207317e-03, -1.9708832e-03, -1.9265555e-03, ...,
         9.8339503e-04, -1.3239905e-03, -9.9254667e-04],
       [-1.7794296e-04,  8.9951250e-04, -1.1194174e-03, ...,
         1.2776342e-03,  6.6736236e-04, -1.1825715e-03],
       ...,
       [ 2.9136413e-03,  2.6662368e-03,  9.9231875e-05, ...,
        -5.4693753e-03,  2.3253520e-03, -2.8927152e-03],
       [-7.2450127e-04,  4.6958555e-03, -1.4273097e-03, ...,
         3.8719655e-03, -1.3461473e-03,  2.1226092e-03],
       [ 2.2172735e-03,  1.1708419e-03, -1.0171951e-03, ...,
         1.0178455e-03, -4.1146367e-04, -2.4446507e-04]], dtype=float32),
                    'w': array([[[-8.27747304e-03, -2.74438374e-02, -2.21472769e-03, ...,
         -8.86141602e-03, -3.90934292e-03, -8.87976680e-03],
        [-4.07242536e-04, -4.85621393e-03,  4.32445714e-03, ...,
         -1.37868756e-02, -1.85714811e-02, -4.08839062e-03],
        [ 1.15496498e-02,  1.84064042e-02,  1.07421465e-02, ...,
          5.71913924e-03, -1.75766507e-03,  1.30740916e-02],
        ...,
        [ 2.03288477e-02, -7.98869040e-03,  2.42463928e-02, ...,
         -5.90799609e-03,  9.65785887e-03, -1.26922876e-02],
        [-6.60760188e-03,  3.74193955e-03, -2.76752817e-03, ...,
         -1.00113945e-02, -5.20520052e-03,  2.33142963e-03],
        [ 4.02665930e-03,  1.88150275e-02,  1.21134911e-02, ...,
          8.95798486e-03, -1.12047391e-02,  1.22876707e-02]],

[[-2.41041579e-03,  7.10168947e-03,  1.04693975e-02, ...,
          4.98504564e-03,  5.78136276e-03,  2.28895564e-02],
        [ 1.42165143e-02, -2.02243053e-03, -2.81853620e-02, ...,
          3.84669926e-04,  1.59979314e-02, -7.58644380e-03],
        [-1.39519563e-02, -3.09518515e-03,  2.12568548e-02, ...,
         -7.30662886e-03,  2.11698143e-03,  2.32529119e-02],
        ...,
        [-4.80401469e-03, -3.60514037e-02,  1.53901596e-02, ...,
         -6.27426151e-03, -2.12004576e-02, -1.42251039e-02],
        [ 3.00109340e-03,  1.31645938e-02, -9.76265408e-03, ...,
          6.94900006e-03, -9.34688654e-03, -7.82344583e-03],
        [ 1.41641488e-02,  1.68120884e-03, -7.23654032e-03, ...,
          7.11144274e-03, -3.88213433e-03,  3.27121117e-03]],

[[ 1.45476218e-02, -9.37034655e-03, -6.78640790e-04, ...,
          8.46901909e-04, -8.01474525e-05,  3.19828419e-03],
        [ 8.89879093e-03, -4.33525816e-03, -1.35370325e-02, ...,
          2.06876174e-03, -6.17312640e-03, -8.82694405e-03],
        [ 2.41799839e-03,  6.58020610e-03,  9.94143449e-03, ...,
         -1.56613183e-03, -2.73650233e-03, -4.65888483e-03],
        ...,
        [ 1.07849902e-02, -1.75305437e-02, -2.42682127e-03, ...,
         -1.34070301e-02, -1.61357895e-02, -5.69711765e-03],
        [-6.43995870e-03, -2.08571344e-03, -1.80199649e-03, ...,
         -7.34707061e-03, -9.09798406e-03,  5.33486251e-03],
        [ 9.34575219e-03,  7.65097933e-03, -1.91252318e-03, ...,
         -4.08657314e-03, -1.32152101e-03,  4.86415019e-03]],

...,

[[ 3.52993794e-03,  4.74431645e-03, -3.68715287e-03, ...,
         -7.07027921e-03, -4.83857328e-03, -3.11120576e-03],
        [-2.01958511e-03, -1.01399682e-02, -1.21086640e-02, ...,
         -9.77886302e-05,  1.80604216e-02,  3.34792165e-03],
        [-5.87149803e-03,  5.73725346e-03,  3.42145655e-03, ...,
         -2.30353465e-03, -1.24953466e-03,  1.07367616e-02],
        ...,
        [ 1.77793801e-04,  8.08947440e-03, -5.54806599e-03, ...,
         -1.89056166e-03,  2.92457291e-03,  7.54432753e-03],
        [ 5.95113123e-03, -5.75119164e-03,  1.05367368e-02, ...,
          6.25454821e-03,  2.01350581e-02, -8.62633158e-03],
        [ 5.85795427e-03, -5.70921367e-03,  2.02379469e-03, ...,
         -1.99293643e-02,  8.57978314e-03,  6.71483949e-03]],

[[-5.95985679e-03, -1.63260717e-02, -2.75160093e-02, ...,
         -1.92538258e-02, -6.87839556e-03, -1.17844520e-02],
        [-1.34677030e-02, -1.00735221e-02, -7.16452254e-03, ...,
          2.05787383e-02, -1.15518575e-03,  1.08702690e-03],
        [ 2.88600903e-02,  8.64205591e-04,  2.54147090e-02, ...,
         -4.79136920e-03,  7.90083129e-03,  1.26480544e-02],
        ...,
        [ 1.99049618e-03,  3.30409640e-03, -2.22005956e-02, ...,
         -1.94110218e-02,  2.99171731e-03, -5.25331218e-03],
        [ 2.87614344e-03, -2.38336977e-02,  2.01235199e-03, ...,
          1.12550538e-02,  2.61978731e-02, -1.17928330e-02],
        [ 1.57598313e-02, -3.94910714e-03,  1.10027734e-02, ...,
          3.87770543e-03,  1.99303608e-02, -9.92721412e-03]],

[[-2.30612271e-02, -2.71627568e-02, -2.97727343e-03, ...,
         -1.52611975e-02, -1.08289560e-02,  4.54727188e-03],
        [-5.37802093e-03,  1.53265456e-02, -7.98696093e-03, ...,
          5.77224419e-03,  3.07626091e-02,  6.27357746e-03],
        [ 7.67199555e-03,  1.04494197e-02, -1.23493625e-02, ...,
          4.24701255e-04, -9.98084806e-03,  5.56567824e-03],
        ...,
        [ 1.42724207e-02,  3.21529619e-02, -1.39995655e-02, ...,
          1.84636272e-03,  3.53092357e-04, -5.84522309e-03],
        [ 1.17971171e-02,  9.07169748e-03, -9.52398684e-03, ...,
          1.18852360e-03,  3.43973120e-03, -7.44402781e-03],
        [ 1.10389059e-02, -1.50724845e-02,  4.58929833e-04, ...,
          1.01573355e-02,  1.08590862e-02,  1.08767196e-03]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.4057414 , 0.42255083, 0.38863888, ..., 0.39593062,
         0.39478564, 0.4200798 ],
        [0.39289957, 0.3691151 , 0.42487112, ..., 0.38257024,
         0.412654  , 0.36031193],
        [0.48402247, 0.4299642 , 0.4866183 , ..., 0.44482148,
         0.4603271 , 0.49831846],
        ...,
        [0.38858014, 0.43790802, 0.44984847, ..., 0.3506014 ,
         0.40318346, 0.35174188],
        [0.41264784, 0.40839374, 0.3950559 , ..., 0.41264805,
         0.42312416, 0.4095264 ],
        [0.44272327, 0.481156  , 0.44693622, ..., 0.40954715,
         0.39949977, 0.39912206]]], dtype=float32)
                }
            }
        },
        'x_layers_5': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': array([-0.0082708 ,  0.00023509, -0.00894445, ..., -0.01141324,
        0.00119758, -0.00033616], dtype=float32)
                    },
                    'linear': {
                        'w': array([[-0.00602971, -0.02676638,  0.0168558 , ..., -0.00994777,
         0.02569861,  0.02104281],
       [-0.03416809,  0.02381924, -0.04968129, ...,  0.03150865,
        -0.00796047,  0.03368018],
       [-0.00179048, -0.03266168,  0.02378024, ...,  0.01224884,
        -0.03469853, -0.00878385],
       ...,
       [ 0.00312726, -0.00156428,  0.00891732, ..., -0.04592262,
        -0.0226953 , -0.04572573],
       [ 0.00380302, -0.03548504, -0.01470138, ..., -0.03882184,
        -0.0209375 ,  0.02290299],
       [-0.01230227, -0.06000502, -0.02844962, ...,  0.00688603,
        -0.00268077, -0.03837435]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.110433 , 1.0006205, 1.0856851, ..., 0.9565266, 1.2328953,
        1.0505819]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': array([ 0.00231809,  0.00047749, -0.00265504, ..., -0.00051469,
        0.00263429,  0.00085238], dtype=float32)
                    },
                    'linear': {
                        'w': array([[ 0.06057307,  0.05345616, -0.02766749, ...,  0.04518164,
        -0.03843925, -0.04467538],
       [ 0.00835851,  0.04028086,  0.03977181, ..., -0.02442977,
         0.0200953 ,  0.02775957],
       [-0.02266712,  0.06793138, -0.04917101, ..., -0.02769876,
        -0.0532935 , -0.02923822],
       ...,
       [ 0.03883331, -0.04628368,  0.01323004, ..., -0.01341316,
         0.04206111,  0.00142196],
       [-0.02760278,  0.02879915,  0.05136937, ..., -0.01124197,
         0.00320576,  0.01919117],
       [-0.03938087, -0.05277129,  0.01173478, ..., -0.00043572,
         0.0437616 , -0.02255625]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.2154577, 1.2388827, 1.1621   , ..., 1.2220094, 1.1810684,
        1.1818546]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': array([-0.00361701,  0.00014588,  0.00192671, ..., -0.00092808,
        0.00158209,  0.00770174], dtype=float32),
                    'scale': array([-0.01588804, -0.01103332, -0.01152071, ..., -0.01402099,
        0.01218372,  0.00757895], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': array([1.0854104, 1.1664495, 1.0141362, ..., 1.0315809, 1.1132778,
       1.0668985], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': array([[ 2.9983949e-03,  4.5695826e-03, -1.4037583e-03, ...,
         2.0471390e-03, -5.5616634e-04, -4.3982812e-03],
       [ 2.6113386e-03, -3.7855271e-03, -1.1430590e-03, ...,
         1.2536233e-03,  5.8888341e-04,  2.4085233e-03],
       [ 6.1706378e-04, -3.5343072e-03,  3.6754673e-03, ...,
        -1.0473594e-03, -2.6400243e-03, -2.6224724e-03],
       ...,
       [-2.7933377e-03,  9.0840273e-04, -8.4660889e-05, ...,
        -4.1713614e-05,  1.6534668e-03,  2.1746625e-04],
       [-1.4082552e-03, -1.5240485e-03,  8.5433078e-04, ...,
        -1.5572939e-04, -8.7523012e-04, -2.2009872e-03],
       [ 1.7080423e-04, -1.5451813e-03, -1.1473884e-03, ...,
        -3.0459089e-03,  3.4824025e-05,  1.3262454e-03]], dtype=float32),
                    'w': array([[[-7.63362879e-03,  5.10084257e-03, -2.08918564e-02, ...,
         -1.14847347e-02,  1.26297809e-02, -2.70505290e-04],
        [ 1.42549560e-03, -2.30036471e-02, -1.78317297e-02, ...,
         -3.46740684e-03, -9.71160643e-03,  1.18957236e-02],
        [ 1.04235457e-02, -1.16101396e-03, -1.41830705e-02, ...,
          1.42017473e-02, -9.16350726e-03, -7.10327178e-03],
        ...,
        [-1.11079877e-02, -2.10176408e-02, -7.89483637e-03, ...,
         -3.19734146e-03,  1.45656744e-03,  8.61223228e-03],
        [-1.26707973e-02,  9.41790640e-03, -1.63667207e-03, ...,
         -3.07305576e-03,  3.62688303e-03,  1.33423358e-02],
        [-2.00318801e-03, -1.57821178e-02,  1.17536416e-04, ...,
         -3.37612652e-03, -1.36024542e-02,  2.33557844e-03]],

[[ 1.45884305e-02,  5.97111415e-03, -3.64374625e-03, ...,
          1.12671806e-02, -1.52111361e-02, -2.27238573e-02],
        [ 6.82519423e-03,  1.92021986e-03, -1.86369689e-05, ...,
         -4.29239916e-03,  1.23106241e-02, -1.37042382e-03],
        [-6.10738993e-03,  1.50299510e-02,  7.28433160e-03, ...,
          2.52019186e-02,  6.20151870e-03, -3.20050726e-03],
        ...,
        [-1.28055811e-02, -1.92083791e-02, -5.68904914e-03, ...,
          6.04085159e-03,  1.54160308e-02, -7.65496399e-03],
        [ 2.93041486e-02, -3.85988294e-03,  9.19975061e-03, ...,
         -1.43612060e-03,  1.01802805e-02, -1.37370424e-02],
        [ 1.66604333e-02,  3.39185516e-03, -6.80400291e-03, ...,
         -2.91282758e-02,  2.17836350e-02, -1.73602868e-02]],

[[ 6.46605762e-03,  1.78333931e-02, -3.17491516e-02, ...,
          2.57964134e-02, -7.77957961e-03, -1.62798669e-02],
        [ 1.54227121e-02,  1.56511215e-03, -2.20833160e-03, ...,
         -1.69989802e-02, -1.93048653e-03,  2.92041781e-03],
        [ 6.90265419e-03,  1.46009345e-02,  1.05491858e-02, ...,
          9.85333323e-03,  1.72033049e-02,  8.98906309e-03],
        ...,
        [-1.71975549e-02,  4.59012063e-03, -2.97780964e-03, ...,
          1.01405531e-02,  1.03456127e-02, -5.95806679e-03],
        [ 6.96400879e-03, -1.21653564e-02,  5.39566530e-03, ...,
         -1.40712578e-02,  2.31189355e-02,  8.29096418e-03],
        [ 7.29186693e-03,  1.95084419e-02,  1.25852050e-02, ...,
          1.14477649e-02, -7.39570195e-03, -1.33903446e-02]],

...,

[[ 1.60712302e-02,  1.72393825e-02, -8.85983370e-03, ...,
          2.25401595e-02,  1.11234747e-03, -1.17539251e-02],
        [ 1.14485109e-03,  6.53832452e-03,  6.59581972e-03, ...,
         -2.94644618e-03,  1.25126634e-02, -7.75689865e-03],
        [ 1.10011222e-02,  1.30102662e-02, -1.72179658e-02, ...,
         -1.87878124e-02,  5.56552643e-03,  6.27677282e-03],
        ...,
        [-1.16724707e-02,  8.18582438e-03, -1.97089836e-02, ...,
         -3.77336703e-02,  2.79105194e-02, -1.06746089e-02],
        [-2.71604154e-02, -5.71717694e-03, -5.00530750e-03, ...,
         -1.93108991e-02,  1.16552832e-03, -4.88433335e-03],
        [ 1.31863188e-02, -6.40873378e-03, -2.03603096e-02, ...,
          2.18288302e-02,  8.27170443e-03,  3.37167783e-03]],

[[ 4.46730107e-03, -1.66290030e-02,  1.62763498e-03, ...,
         -1.12466430e-02,  1.96464509e-02,  1.92972664e-02],
        [ 2.34295800e-02, -2.30623279e-02, -1.84654929e-02, ...,
          2.66797300e-02, -9.05167684e-03,  1.73226688e-02],
        [-1.98417939e-02, -4.87846648e-03,  1.43612027e-02, ...,
         -1.52964760e-02, -8.41074437e-03, -1.12230908e-02],
        ...,
        [ 5.46930358e-03, -6.18861243e-03,  1.16656190e-02, ...,
          1.69505365e-02, -1.48189003e-02,  1.05194906e-02],
        [ 9.57203098e-03, -1.99217685e-02, -1.48693305e-02, ...,
         -6.96395757e-03,  1.71882864e-02, -1.06076542e-02],
        [ 2.47014947e-02,  7.35677918e-03,  1.64079368e-02, ...,
         -2.86943242e-02, -2.19791047e-02, -2.81189289e-02]],

[[ 1.29230488e-02, -1.24336779e-02,  1.27518242e-02, ...,
         -1.73731763e-02, -2.01447140e-02, -2.70947861e-03],
        [-1.85279734e-02,  4.07904806e-03, -2.91116349e-03, ...,
          3.54991818e-04, -1.13391196e-02,  1.60872890e-03],
        [ 6.38186233e-03,  6.76335953e-03,  1.04324734e-02, ...,
          1.50489118e-02, -5.08294720e-03, -6.67493325e-03],
        ...,
        [ 2.76850406e-02,  3.05225533e-02,  1.50380582e-02, ...,
          1.40753342e-02, -1.41735123e-02,  2.10294109e-02],
        [-3.27004190e-03,  6.93722046e-04, -1.51707726e-02, ...,
          3.33688594e-02, -3.51021794e-04,  2.66710799e-02],
        [-9.72767826e-03, -3.45314061e-03, -8.60371813e-03, ...,
         -3.05569042e-02,  2.07188912e-02, -1.41795063e-02]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.41908428, 0.46363786, 0.51578134, ..., 0.4705185 ,
         0.43959528, 0.38401735],
        [0.52108026, 0.35619184, 0.4665575 , ..., 0.6334982 ,
         0.3458498 , 0.3469979 ],
        [0.45826864, 0.42733502, 0.4562944 , ..., 0.48786226,
         0.3798276 , 0.39579493],
        ...,
        [0.48633897, 0.49172735, 0.4520785 , ..., 0.44899756,
         0.46653658, 0.40077436],
        [0.61393267, 0.48808634, 0.46364224, ..., 0.4385616 ,
         0.41498303, 0.53538525],
        [0.57095027, 0.4214728 , 0.4768966 , ..., 0.563258  ,
         0.52000856, 0.48005775]]], dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': array([0.13068558, 0.11264031, 0.11708908, 0.12530498, 0.12238726,
       0.13479206, 0.0961272 , 0.10192457, 0.06898148, 0.11135861,
       0.11440782, 0.11942353, 0.11138015, 0.11427903, 0.15602514,
       0.10358352, 0.12103997, 0.10196049, 0.11657108, 0.1105193 ,
       0.13546531, 0.10212965, 0.12147649, 0.11892303, 0.14143457,
       0.13192987, 0.12624545, 0.12940401, 0.11032961, 0.11966604,
       0.1251902 , 0.12702945, 0.12194424, 0.12021966, 0.10040382,
       0.09689584, 0.13449894, 0.145295  , 0.12237985, 0.1316158 ,
       0.14289606, 0.1149406 , 0.12861389, 0.09718668, 0.10809228,
       0.11319933, 0.1056805 , 0.11801772, 0.13401496, 0.13912912,
       0.1070198 , 0.10733105, 0.12350067, 0.12399949, 0.12067261,
       0.09138734, 0.1365333 , 0.1002665 , 0.13340193, 0.12104555,
       0.11990233, 0.10155478, 0.1109368 , 0.12974575, 0.11362689,
       0.13002796, 0.13385442, 0.1020591 , 0.10243788, 0.12618516,
       0.13293782, 0.10212671, 0.0935626 , 0.12699246, 0.14163223,
       0.1277229 , 0.11770076, 0.14909612, 0.10548238, 0.09269526],
      dtype=float32)
                },
                'post': {
                    'b': array([ 0.00065614,  0.00162599,  0.00121886, ..., -0.00042695,
        0.00442083,  0.00376454], dtype=float32),
                    'w': array([[[-2.49264226e-03,  1.17608486e-02,  1.48023441e-02, ...,
         -1.27285868e-02,  4.40729549e-03,  1.67888775e-02],
        [ 1.92789957e-02, -4.13750857e-03, -1.37377379e-03, ...,
         -8.48557975e-04, -3.75018991e-03,  1.16395194e-03],
        [-4.72128903e-03,  2.33182032e-02,  3.60964076e-03, ...,
         -2.32205801e-02,  4.94472077e-03,  1.40123470e-02],
        ...,
        [ 1.15733771e-02,  5.74059924e-03, -3.51108727e-03, ...,
          3.92256508e-04, -5.45613840e-03, -6.04295032e-03],
        [ 1.52369856e-03,  1.14200599e-02,  1.37395766e-02, ...,
         -2.21631420e-03,  1.10637462e-02,  1.66783351e-02],
        [ 2.96310778e-03,  1.01144752e-02,  1.01158349e-02, ...,
         -3.47329327e-03, -1.06788361e-02, -9.48280375e-03]],

[[ 1.18467724e-03,  9.04635340e-03,  8.45457835e-04, ...,
          7.94761814e-03,  7.34839961e-03, -2.27858010e-03],
        [ 1.66778546e-03,  6.13414776e-03,  1.14113288e-02, ...,
          9.64034069e-03, -3.07440711e-03,  4.02209256e-03],
        [ 8.61036871e-03,  1.73594467e-02,  1.19628226e-02, ...,
         -9.06090625e-03,  8.20799638e-03, -8.09912384e-03],
        ...,
        [-1.25589455e-02,  5.86901745e-03,  2.30502896e-02, ...,
          6.20257668e-03,  3.07463366e-03, -6.61631068e-03],
        [ 3.31885577e-03, -1.86445168e-03,  6.54553808e-03, ...,
         -4.83053364e-03, -1.42367510e-02,  4.67196153e-03],
        [-1.57865193e-02,  1.36779537e-02,  1.20302830e-02, ...,
          1.76998470e-02, -1.40256416e-02, -1.53108295e-02]],

[[-1.27371103e-02,  7.25474674e-03,  2.99782772e-03, ...,
         -4.67289146e-03,  9.70524736e-03, -9.79083963e-03],
        [-9.33346525e-03,  1.18548591e-02,  1.21941520e-02, ...,
         -1.36308419e-03,  5.52827027e-04, -4.57338477e-03],
        [-4.29865718e-03,  1.51828155e-02,  1.31705580e-02, ...,
          1.54210422e-02, -2.56938604e-03, -2.26507578e-02],
        ...,
        [-1.19850691e-02, -5.04698977e-03,  4.24150552e-04, ...,
          2.81316243e-05, -2.94065755e-03,  9.14281607e-03],
        [-4.26020054e-03, -1.25661027e-02,  6.52692886e-03, ...,
         -7.43539585e-03, -1.07485242e-02,  2.09866418e-03],
        [-9.22300294e-03,  8.05429090e-03, -1.09433290e-02, ...,
         -2.22045872e-02, -7.96054862e-03, -2.69814506e-02]],

...,

[[-1.74979214e-03,  1.53169855e-02,  1.45188738e-02, ...,
         -1.04527380e-02,  5.94762387e-03,  1.58894584e-02],
        [ 1.14164094e-03,  9.81054828e-03, -8.63398134e-04, ...,
          1.48538472e-02,  4.86827781e-03,  2.16232310e-03],
        [ 3.80162266e-03, -8.46225303e-03, -5.80415549e-03, ...,
          1.65113118e-02, -1.06529752e-02, -1.71929523e-02],
        ...,
        [-3.90411704e-03,  1.42602506e-03,  1.33081507e-02, ...,
         -1.65443253e-02,  1.35355191e-02, -1.18037630e-02],
        [-1.06663369e-02,  7.06519792e-03,  1.17162531e-02, ...,
          6.70785643e-03,  1.88109861e-05, -8.13740771e-03],
        [-1.54220717e-04,  7.49557326e-03, -8.56330432e-03, ...,
          1.53965540e-02,  3.06257210e-03, -1.74533334e-02]],

[[-2.15597311e-03, -6.33097254e-03, -1.68540434e-03, ...,
          7.62878265e-03, -2.14439426e-02, -6.04961952e-03],
        [ 9.19801823e-04, -8.28042254e-03,  1.12024667e-02, ...,
         -5.16699906e-03,  1.20820384e-02, -8.03519040e-03],
        [ 1.63801871e-02,  1.07365102e-02,  8.78750999e-03, ...,
         -1.35056078e-02, -2.12804507e-03,  1.64057296e-02],
        ...,
        [ 1.10089932e-04, -5.69088431e-03, -8.44818167e-03, ...,
          9.23904590e-03, -2.86108758e-02, -6.02929480e-03],
        [-7.13141588e-03,  1.17807910e-02,  3.69701535e-03, ...,
         -8.11086036e-03,  1.21025126e-02, -8.65382981e-03],
        [ 3.45807686e-03,  6.63782144e-03, -1.05535472e-02, ...,
         -2.80095427e-03, -5.49800182e-03, -1.31416628e-02]],

[[ 5.33286156e-03,  4.65261610e-03, -4.57288744e-03, ...,
         -1.17293769e-03,  1.02246245e-02,  3.17146722e-03],
        [ 1.14073955e-04, -8.93527269e-03,  1.22774597e-02, ...,
         -1.29055474e-02,  1.76185332e-02, -5.55372750e-03],
        [ 8.42548721e-03,  7.63535034e-03,  5.08680940e-03, ...,
          1.00998925e-02,  5.60012460e-03,  3.20164138e-03],
        ...,
        [-1.29430825e-02, -1.70630645e-02,  6.33569725e-04, ...,
         -2.31857924e-03, -4.24584793e-03, -1.89849064e-02],
        [-1.04850493e-02,  8.94758850e-03,  3.43736378e-03, ...,
         -9.55532677e-03, -5.64501993e-03,  1.00068869e-02],
        [ 1.50155732e-02,  6.59107976e-03,  2.04607006e-02, ...,
         -6.30389946e-03,  1.74301548e-03,  6.87498460e-03]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.35763955, 0.34659582, 0.32721144, ..., 0.33384556,
         0.37323347, 0.3346946 ],
        [0.3520102 , 0.34894353, 0.33540943, ..., 0.35193616,
         0.3715671 , 0.3580485 ],
        [0.34773424, 0.3472333 , 0.34598884, ..., 0.3578927 ,
         0.3603485 , 0.35333636],
        ...,
        [0.38140067, 0.407084  , 0.36061957, ..., 0.39202622,
         0.36132824, 0.36536968],
        [0.36460268, 0.35297278, 0.34932053, ..., 0.3619079 ,
         0.35416538, 0.3620825 ],
        [0.3571164 , 0.3770913 , 0.37917823, ..., 0.3653027 ,
         0.35700715, 0.36384162]]], dtype=float32)
                },
                'query': {
                    'b': array([[-0.00535969,  0.00544131,  0.00895877, ..., -0.01104905,
        -0.00018965, -0.00259527],
       [-0.00518589,  0.0229747 ,  0.03777637, ...,  0.00089205,
         0.01973232, -0.03037073],
       [ 0.00032148,  0.01091193, -0.01324205, ..., -0.00556419,
         0.01459017,  0.01027916],
       ...,
       [-0.02908402, -0.00352269, -0.01038569, ..., -0.02177537,
         0.02019865, -0.00165102],
       [-0.00203704, -0.00681012,  0.00786141, ..., -0.01706576,
         0.01285954, -0.00309793],
       [-0.00697265, -0.01840293, -0.01635662, ...,  0.00518259,
        -0.01693589,  0.00704248]], dtype=float32),
                    'w': array([[[-3.59171070e-03,  4.86029225e-04,  1.73513964e-02, ...,
         -2.03544274e-02, -7.29639363e-03, -1.37512004e-02],
        [ 3.51476576e-03, -4.14279569e-03,  5.83695527e-03, ...,
          4.22853889e-04,  1.60983242e-02,  1.52249995e-03],
        [ 1.19537720e-03,  8.63350928e-03, -1.61050875e-02, ...,
          1.76351350e-02, -1.06409669e-03, -1.31810606e-02],
        ...,
        [ 5.92035335e-03, -5.20166615e-03, -9.49679222e-03, ...,
          2.86339456e-03,  8.24203063e-03,  7.12555787e-03],
        [-1.71734393e-02, -2.81062094e-04, -9.40955617e-03, ...,
         -1.18391523e-02,  3.19176801e-02,  2.29466725e-02],
        [-1.70886412e-03,  9.49707627e-03, -2.41508223e-02, ...,
          3.97554711e-02, -1.66079821e-03, -4.81888931e-03]],

[[-1.46383438e-02,  1.41944103e-02,  5.67111326e-03, ...,
         -9.32640955e-03,  8.63393396e-03,  1.00248307e-02],
        [ 6.26494223e-03,  3.16732476e-04, -1.40951909e-02, ...,
          6.06844062e-03,  1.06160063e-02, -1.06019462e-02],
        [-4.10887972e-03,  2.03521829e-02, -1.48970485e-02, ...,
          3.96736488e-02,  3.87813285e-04,  3.07012890e-02],
        ...,
        [ 1.83751639e-02, -1.43249510e-02, -8.15660984e-04, ...,
          2.11314317e-02,  6.82663219e-03, -8.28873739e-03],
        [-1.46689964e-02,  1.26939155e-02,  2.44758632e-02, ...,
          3.65973674e-02,  4.20098566e-03,  2.69141011e-02],
        [-1.48770250e-02,  5.21388464e-03,  3.18557769e-02, ...,
         -3.49426009e-02,  4.44804039e-03,  4.32176562e-03]],

[[ 2.88298596e-02,  9.32741538e-03,  1.72832813e-02, ...,
          4.46523278e-04,  2.68981513e-03,  5.98374987e-03],
        [ 2.40889285e-02,  8.43146723e-03,  3.40014920e-02, ...,
          1.70410480e-02,  1.47562288e-02, -1.40717747e-02],
        [ 2.41849162e-02,  4.18936312e-02, -1.43234897e-02, ...,
         -2.23203795e-03,  1.00070983e-02,  4.57092514e-03],
        ...,
        [ 9.80064739e-03,  1.37837976e-02, -1.54846758e-02, ...,
         -7.30299065e-03, -2.34902333e-02,  2.49213483e-02],
        [-2.40328405e-02,  5.64369373e-04,  2.47111004e-02, ...,
          1.14819547e-02,  5.77154988e-03,  1.32483197e-02],
        [ 9.83753853e-05, -9.01767798e-03,  6.56622089e-03, ...,
          2.23362492e-03, -2.11462704e-03,  1.57479942e-02]],

...,

[[ 5.40862512e-03, -3.01395776e-03, -1.02795791e-02, ...,
         -2.04675052e-05,  6.88017486e-03,  6.62454125e-03],
        [ 2.69270334e-02,  5.93627628e-04,  7.10049644e-03, ...,
          1.10973520e-02,  1.44305639e-02,  6.75332267e-03],
        [ 1.68211143e-02,  1.43166734e-02, -1.70226805e-02, ...,
          5.40420366e-03, -6.13971939e-03,  5.04455995e-03],
        ...,
        [-1.04959682e-02,  1.22499270e-02, -3.98181472e-03, ...,
         -2.50086319e-02,  1.79599933e-02,  8.63910187e-03],
        [-4.82433289e-03,  4.81557893e-03,  1.15773054e-02, ...,
         -2.67953193e-03,  1.36260930e-02, -3.19654378e-03],
        [ 9.28092562e-03, -1.06045241e-02, -2.29395349e-02, ...,
          2.52178833e-02, -1.72531349e-04,  4.21606936e-02]],

[[-2.50803586e-02, -4.01995191e-03,  3.31264175e-02, ...,
         -2.45887712e-02,  5.99560980e-03,  2.98948269e-02],
        [-7.43411994e-03,  1.30056199e-02,  4.40202840e-02, ...,
          7.55285705e-03,  2.05392800e-02, -2.42556576e-02],
        [-8.18122365e-03,  1.51081453e-03,  4.04688111e-03, ...,
          2.66218162e-03, -9.52452712e-04,  9.29485261e-03],
        ...,
        [-1.86852179e-02,  4.81882095e-02, -1.76106952e-02, ...,
         -1.41204605e-02, -2.12881453e-02,  1.05350628e-03],
        [ 1.89235527e-02, -3.39181833e-02, -6.47730986e-03, ...,
         -1.87814294e-03,  3.30813713e-02,  4.31669410e-03],
        [-2.97957305e-02, -6.02693250e-03,  1.85229424e-02, ...,
         -8.48276727e-03, -2.81591713e-02,  2.00134758e-02]],

[[-1.08893197e-02,  2.01132782e-02, -1.41311232e-02, ...,
          4.49838080e-02,  1.88883208e-02, -2.49197595e-02],
        [ 1.31158363e-02, -1.01136826e-02, -1.65932309e-02, ...,
          2.25598309e-02,  1.38090095e-02,  1.44756946e-03],
        [-2.09184662e-02, -4.16760426e-03, -1.00047747e-02, ...,
         -2.33833566e-02, -2.47874651e-02, -1.74675826e-02],
        ...,
        [ 1.01681864e-02,  2.75148526e-02, -3.07024946e-03, ...,
         -1.39357764e-02, -2.08011828e-02,  2.21577492e-02],
        [-3.29795070e-02,  7.55552202e-04, -3.07305008e-02, ...,
         -6.28283992e-03, -1.76763702e-02,  9.68733989e-03],
        [-2.69406033e-03,  4.98209661e-03,  4.78894822e-03, ...,
         -4.48810458e-02, -1.16679259e-02, -1.80438738e-02]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.50367874, 0.53149545, 0.59063506, ..., 0.59299475,
         0.51440156, 0.52862114],
        [0.557544  , 0.45456147, 0.58691275, ..., 0.61052394,
         0.527172  , 0.4741048 ],
        [0.57196194, 0.5562834 , 0.578642  , ..., 0.59484315,
         0.4918774 , 0.5158486 ],
        ...,
        [0.7786503 , 0.71618634, 0.53932446, ..., 0.67756075,
         0.6170942 , 0.63507503],
        [0.47351697, 0.5721508 , 0.661362  , ..., 0.643544  ,
         0.6517246 , 0.5195119 ],
        [0.6562377 , 0.811682  , 0.80010235, ..., 0.74999005,
         0.6152983 , 0.7746392 ]]], dtype=float32)
                },
                'value': {
                    'b': array([[ 1.7673090e-04, -2.0610308e-03, -1.2899771e-03, ...,
         1.7491290e-03, -1.8086215e-04, -4.2822282e-04],
       [-1.4554589e-03, -2.9118320e-03,  3.2678701e-05, ...,
         2.7426528e-03,  1.5872034e-04, -2.5911438e-03],
       [ 2.2816991e-04, -3.2166156e-03, -7.0322072e-04, ...,
        -1.3632405e-03,  1.9719843e-03, -6.0191058e-04],
       ...,
       [ 2.8800408e-03,  1.0955781e-03, -1.1744021e-03, ...,
         2.8533933e-03, -4.6616411e-03, -2.5245901e-03],
       [ 3.6332270e-03, -3.3154138e-03, -9.8116521e-04, ...,
        -7.3912339e-03, -1.3474902e-03, -3.7023891e-03],
       [ 1.6156649e-03, -1.6597542e-03,  1.2324060e-03, ...,
        -3.4989812e-04, -7.1168086e-04, -4.4309464e-04]], dtype=float32),
                    'w': array([[[-1.94498431e-02, -7.64790457e-03, -6.29008841e-03, ...,
         -5.54918498e-03,  2.08781715e-02, -8.77577718e-03],
        [ 1.24037508e-02, -7.79988663e-03,  1.69514921e-02, ...,
          1.23843411e-02,  1.49343666e-02, -8.53807293e-03],
        [ 1.97821800e-02, -4.14963113e-03,  1.94442533e-02, ...,
          1.06186885e-02,  2.99014500e-03, -1.60769522e-02],
        ...,
        [-7.28181889e-03, -2.31254287e-02,  1.61775816e-02, ...,
          5.33160521e-03,  6.89735729e-03,  3.29807326e-02],
        [ 6.98302267e-03,  4.92673740e-03, -1.29372906e-02, ...,
          1.28492406e-02, -1.33066913e-02,  1.97734907e-02],
        [ 6.50113123e-03,  5.44579467e-03, -1.27206072e-02, ...,
          2.45580599e-02, -2.19962932e-02, -1.54230632e-02]],

[[-1.77739859e-02, -5.50679024e-03, -8.38629995e-03, ...,
         -4.09661606e-03, -7.11583626e-03, -3.54071904e-04],
        [-7.72978133e-03, -1.64529793e-02, -2.24194936e-02, ...,
          1.31929736e-03, -5.21678757e-03, -2.74662077e-02],
        [-4.47138958e-03, -2.48589665e-02, -3.71789970e-02, ...,
         -1.37483077e-02,  1.98139921e-02,  1.00762257e-02],
        ...,
        [ 1.82423387e-02,  7.20844930e-03, -4.16023453e-04, ...,
          6.95282593e-03, -1.09914485e-02,  3.27756591e-02],
        [-1.78071707e-02,  1.22669460e-02,  1.59574244e-02, ...,
         -1.26053896e-02, -1.76442147e-03,  9.96018015e-03],
        [-1.38700230e-03,  1.85048878e-02,  2.08969507e-03, ...,
          2.21961155e-03,  1.09072286e-03,  1.97160766e-02]],

[[-1.44777093e-02, -1.02259917e-02, -2.36834935e-03, ...,
          4.28947667e-03,  4.29532304e-03,  1.00072259e-02],
        [ 1.35756016e-03,  2.19851313e-03,  3.75138666e-03, ...,
         -4.37067077e-03, -2.91547715e-03, -1.73489132e-03],
        [-1.01620359e-02, -2.68569915e-03, -4.93878452e-03, ...,
          5.23231784e-03, -9.10644667e-05, -3.16470279e-03],
        ...,
        [-7.41225248e-03,  1.03747193e-02,  1.48918377e-02, ...,
          1.07995542e-02, -2.21234700e-03, -3.90553102e-03],
        [ 5.15888561e-04, -1.18454499e-02,  2.06145691e-03, ...,
         -3.89568438e-03, -5.59155783e-03,  1.84259855e-03],
        [ 5.13159391e-03, -8.13061558e-03,  9.86621366e-04, ...,
          1.12553816e-02, -1.75013253e-03, -2.49404082e-04]],

...,

[[-3.74985946e-04, -7.54384324e-03, -1.71769522e-02, ...,
          4.92169242e-03,  4.84458543e-03, -1.07520698e-02],
        [-6.80191861e-03, -6.68954849e-03,  7.05898274e-03, ...,
         -2.51647970e-03, -6.80687372e-03,  8.08799174e-03],
        [-1.05781052e-02, -1.18761417e-02,  9.32055712e-03, ...,
         -3.77121754e-03, -8.92526191e-03, -1.82561213e-04],
        ...,
        [ 6.99557597e-03,  3.04202898e-03,  2.79681524e-03, ...,
         -3.84832243e-03, -4.28989576e-03,  1.55171445e-02],
        [-4.44741081e-03,  3.59940785e-03,  3.78308492e-03, ...,
         -3.47161149e-05,  9.30958800e-03, -6.13607350e-04],
        [ 7.34001398e-03, -1.51855277e-03, -4.64390637e-03, ...,
         -1.20918099e-02,  5.04805939e-03,  3.69474082e-03]],

[[ 5.32482238e-03,  1.99842174e-02,  7.88905192e-03, ...,
         -1.52313486e-02,  1.57131162e-02, -6.79284101e-03],
        [ 7.40573695e-03,  1.68471728e-02,  1.28021762e-02, ...,
         -3.64350784e-03,  7.36924727e-03,  9.20176227e-03],
        [-3.09012793e-02, -5.86587796e-03,  3.41506931e-03, ...,
          2.07649395e-02,  1.33394487e-02, -1.88462976e-02],
        ...,
        [ 2.61604469e-02, -6.53135497e-03, -8.68466869e-03, ...,
          1.76735763e-02,  3.43106836e-02,  1.20292762e-02],
        [-1.84630032e-03, -2.77672266e-03, -3.62162702e-02, ...,
          2.32456042e-03,  6.98588556e-03, -3.98799870e-03],
        [-1.34493820e-02,  1.02754366e-02,  1.87521502e-02, ...,
          2.15228759e-02, -7.09294761e-03, -1.88284277e-04]],

[[-1.30176395e-02,  1.15965446e-02, -1.36257289e-03, ...,
          1.13581447e-02,  8.71351361e-03, -6.23817975e-03],
        [ 5.81024494e-03,  1.62437931e-02, -5.57216350e-04, ...,
         -2.28420808e-03,  2.54431181e-03,  9.41395701e-04],
        [ 2.17519812e-02, -1.07939215e-02, -8.15818086e-03, ...,
         -1.20645314e-02,  1.17858164e-02,  2.23123282e-02],
        ...,
        [-3.23988474e-03,  4.68354439e-03, -8.89800303e-03, ...,
         -1.05521753e-02,  6.07082248e-03,  7.68872537e-03],
        [-6.99350843e-03,  2.20081629e-03, -1.68537572e-02, ...,
          3.71227157e-03, -5.02140680e-03,  2.17879447e-03],
        [-8.09752475e-03, -3.74472264e-04,  1.37547473e-03, ...,
         -9.78743657e-03,  1.00383982e-02,  8.58860463e-03]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.4139829 , 0.38582364, 0.36922434, ..., 0.38812888,
         0.43827474, 0.3725472 ],
        [0.43202645, 0.43002877, 0.3970315 , ..., 0.4287623 ,
         0.4383182 , 0.42981267],
        [0.42699966, 0.40253747, 0.43004003, ..., 0.42076588,
         0.44828245, 0.41572842],
        ...,
        [0.45372197, 0.47391298, 0.43469998, ..., 0.4835692 ,
         0.42743284, 0.46932158],
        [0.43468997, 0.42674968, 0.42417058, ..., 0.42669776,
         0.42347258, 0.4466936 ],
        [0.42011386, 0.43092752, 0.44714668, ..., 0.4106344 ,
         0.42258897, 0.41634768]]], dtype=float32)
                }
            }
        },
        'x_layers_6': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': array([-0.05319161, -0.00180769, -0.00665944, ..., -0.00591564,
       -0.01293479, -0.01065589], dtype=float32)
                    },
                    'linear': {
                        'w': array([[-0.13107388, -0.00877222, -0.01354472, ...,  0.00660291,
         0.00827086,  0.06084311],
       [-0.08909545, -0.00488692, -0.04813811, ..., -0.01833421,
        -0.02547176, -0.04040889],
       [-0.03659885, -0.00826377, -0.00942399, ..., -0.01570849,
        -0.0093856 , -0.01922558],
       ...,
       [-0.0066282 , -0.03163596, -0.01922997, ...,  0.00631564,
        -0.0245108 , -0.03882673],
       [-0.01133524, -0.01882924, -0.01039014, ..., -0.02043537,
         0.06204742,  0.03220206],
       [-0.05412352,  0.02532915,  0.02422992, ..., -0.03567279,
        -0.02308617,  0.01022565]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[2.1763372, 1.0366181, 1.0428822, ..., 1.0158404, 1.0948536,
        0.9894584]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': array([ 0.00293988,  0.0009932 ,  0.00107772, ..., -0.00040506,
        0.0001442 ,  0.00212154], dtype=float32)
                    },
                    'linear': {
                        'w': array([[-0.4611281 , -0.20447981, -0.2977535 , ...,  0.23038045,
         0.46599442, -0.1837977 ],
       [ 0.0019854 ,  0.006754  , -0.01749402, ..., -0.05501841,
         0.04297751,  0.01049735],
       [ 0.02726045,  0.03819789,  0.0172031 , ..., -0.00300778,
        -0.02944737,  0.00064869],
       ...,
       [-0.06164307, -0.00551394, -0.02048076, ..., -0.00060017,
        -0.01957849, -0.02914051],
       [-0.02144488,  0.02477437, -0.01749869, ...,  0.00511124,
         0.00741991,  0.02064732],
       [-0.02758567,  0.0044741 , -0.01848485, ..., -0.02339551,
         0.01778207, -0.0161617 ]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.5820653, 1.3840346, 1.8315593, ..., 1.5262436, 1.5674025,
        1.4928457]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': array([ 0.0018998 ,  0.00821097, -0.00399487, ..., -0.00671882,
        0.00488133,  0.00149973], dtype=float32),
                    'scale': array([-0.01244817,  0.00892163, -0.01021829, ..., -0.02017842,
       -0.01239914,  0.01176315], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': array([1.1080961, 1.1210177, 1.0306809, ..., 1.050785 , 1.0788523,
       1.0700463], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': array([[-3.1537301e-04, -1.3355116e-03, -6.2528707e-04, ...,
        -1.7741041e-03,  2.3211220e-04, -1.1813914e-03],
       [-1.0428097e-03, -3.3338604e-04,  1.5919108e-03, ...,
        -1.4696107e-03,  1.1327536e-03,  2.0284744e-03],
       [ 2.1186552e-03, -1.5467203e-04, -5.1601962e-03, ...,
        -1.9826847e-03,  1.6850738e-04, -4.3730572e-04],
       ...,
       [-8.5889758e-04, -4.9279694e-04, -4.7004360e-04, ...,
         8.0387533e-04, -8.7275222e-04,  1.2364115e-03],
       [-1.6918234e-03, -1.5437314e-03,  1.1954332e-03, ...,
        -7.0258800e-05,  2.0370714e-03,  1.2066830e-03],
       [ 1.9361990e-03,  2.2996487e-03, -1.3164624e-04, ...,
        -4.9002131e-04, -5.7391531e-04,  1.3540803e-03]], dtype=float32),
                    'w': array([[[ 2.24064197e-02, -5.44897374e-03, -7.02082319e-03, ...,
         -1.11840591e-02,  5.14022214e-03,  4.75016190e-03],
        [ 2.71372520e-03, -1.44458525e-02, -1.66786220e-02, ...,
          2.48380173e-02,  5.30734845e-03, -4.04858962e-03],
        [-2.02985518e-02, -2.22432688e-02,  5.04163140e-03, ...,
          1.05001461e-02, -5.45279123e-03,  1.94659010e-02],
        ...,
        [-5.60708297e-03,  3.33053153e-03,  5.22669929e-04, ...,
          8.10892379e-05,  1.56823394e-03,  8.21985491e-03],
        [-3.26173590e-03, -9.37842578e-03, -2.09067836e-02, ...,
          3.57581908e-03,  1.43249426e-02,  2.75801085e-02],
        [-9.59258806e-03, -6.47648470e-03,  3.48970410e-03, ...,
          7.60874245e-03,  1.22080911e-02, -2.02672509e-03]],

[[-1.95381790e-02, -5.11802733e-03,  2.39367802e-02, ...,
          3.54355387e-02,  1.34878941e-02, -1.43186683e-02],
        [-3.03459074e-03,  2.49964036e-02,  1.72070097e-02, ...,
         -8.71973764e-03, -8.63747112e-03, -4.67304001e-03],
        [ 2.06037108e-02,  1.21650053e-03,  1.40657285e-02, ...,
         -1.10337213e-02, -7.38372421e-03, -2.61721238e-02],
        ...,
        [-5.49486722e-04, -1.81620196e-03,  1.72393788e-02, ...,
          1.27927894e-02, -1.27091613e-02, -2.18460225e-02],
        [ 2.82634841e-03, -4.17095469e-03,  5.61080221e-03, ...,
         -2.99666164e-04, -3.12445220e-03,  4.93399007e-03],
        [ 4.52524470e-03,  9.54926945e-03, -7.29968958e-03, ...,
          3.97754135e-03,  9.44443140e-03,  1.27538126e-02]],

[[-5.62925544e-03,  8.57002381e-03, -1.84838492e-02, ...,
          7.16796285e-03,  4.38968139e-03,  1.79966574e-03],
        [ 1.19244149e-02,  1.07788704e-02, -1.35443378e-02, ...,
          3.04871402e-03, -6.43102033e-03,  6.66176388e-03],
        [ 3.98526574e-03, -1.05916206e-02,  2.96695288e-02, ...,
         -5.51141705e-03,  4.54244250e-03, -1.82737987e-02],
        ...,
        [-6.61038002e-03,  2.79864483e-03, -7.08234403e-03, ...,
          3.83795425e-03, -2.96489871e-03, -5.39579894e-03],
        [ 3.26566026e-02,  8.46816693e-03, -1.09205442e-03, ...,
          3.56886513e-03, -1.85176712e-02,  1.10904071e-02],
        [ 9.65723861e-03, -3.45743331e-03, -8.87637306e-03, ...,
         -2.56780703e-02,  2.16682106e-02,  1.02111697e-02]],

...,

[[-3.23573835e-02,  1.58385504e-02, -1.93643328e-02, ...,
         -1.63480826e-02,  1.43160252e-03, -2.54204497e-04],
        [-1.46767963e-02,  9.15251300e-03, -9.66769084e-03, ...,
          1.35026099e-02,  1.34027656e-03, -9.63582471e-03],
        [ 2.32933369e-03, -8.69661756e-03,  3.51569057e-03, ...,
          4.57025785e-03,  1.66973900e-02,  5.13487589e-03],
        ...,
        [-1.03506735e-02,  4.51176055e-03,  1.00738043e-02, ...,
          4.27281018e-03, -7.19336187e-03,  2.39671059e-02],
        [-4.19330876e-03, -7.04603561e-04,  3.40670459e-02, ...,
          5.25396783e-03,  7.08937878e-03, -1.45133920e-02],
        [ 4.01711185e-03, -8.24975781e-03, -1.28848003e-02, ...,
          2.05915328e-03,  2.27072136e-03, -1.37904787e-03]],

[[-1.11920116e-02,  1.36633022e-02, -7.92877097e-03, ...,
          5.84112387e-03,  1.75850641e-03, -1.12324180e-02],
        [ 8.76821671e-03,  1.13930833e-02, -1.30911246e-02, ...,
          4.94425278e-03, -1.40145589e-02,  2.47357897e-02],
        [-9.64955520e-03, -4.49962169e-03, -1.20574422e-02, ...,
         -2.14102063e-02,  1.00340284e-02, -2.76313233e-03],
        ...,
        [ 1.69542097e-02, -1.55539801e-02, -1.63793787e-02, ...,
         -1.13881603e-02,  1.44188926e-02, -4.21228483e-02],
        [-7.36775529e-03,  5.71771106e-03, -7.00307777e-03, ...,
          4.74041421e-03, -1.91577543e-02,  3.47967888e-03],
        [ 1.38436770e-03,  5.85974287e-03,  1.39302909e-02, ...,
         -9.31549631e-03, -8.43442883e-03, -1.25797209e-03]],

[[-4.28056484e-03, -1.93030061e-03,  4.81113559e-03, ...,
          4.17014398e-03, -7.71327922e-03, -1.40038319e-02],
        [ 1.41333453e-02, -8.93208664e-03, -8.02380871e-03, ...,
          7.98764359e-03,  5.41705173e-03,  1.27402926e-02],
        [-1.76554769e-02, -2.69824755e-03,  1.54527044e-02, ...,
          1.34667978e-02,  6.39165938e-03,  9.21813771e-03],
        ...,
        [ 2.51757596e-02,  4.08009719e-03, -1.52747296e-02, ...,
         -2.46939585e-02,  1.07817482e-02, -2.32611988e-02],
        [-2.06047632e-02,  9.03889630e-03, -1.09267971e-02, ...,
          3.04490415e-04, -3.37161683e-03,  3.82427089e-02],
        [ 4.00011020e-04,  2.06485973e-03, -9.59498808e-03, ...,
         -6.34575868e-03,  2.44500060e-02,  1.71611514e-02]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.5237434 , 0.51841205, 0.49563366, ..., 0.51283103,
         0.42288467, 0.4275481 ],
        [0.5076875 , 0.47348517, 0.43997252, ..., 0.50296605,
         0.4526671 , 0.43753073],
        [0.4770098 , 0.49414268, 0.5219133 , ..., 0.45808694,
         0.522959  , 0.6035626 ],
        ...,
        [0.3992194 , 0.51330894, 0.46458468, ..., 0.43114203,
         0.4407429 , 0.46398616],
        [0.47514358, 0.35731274, 0.46516797, ..., 0.3387543 ,
         0.36571157, 0.49621606],
        [0.4177364 , 0.40681756, 0.54289186, ..., 0.430469  ,
         0.45580944, 0.39901212]]], dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': array([0.11642151, 0.08755226, 0.13129562, 0.11135118, 0.1541929 ,
       0.14946258, 0.16739756, 0.07969241, 0.07921032, 0.10419153,
       0.1310537 , 0.09849536, 0.12754586, 0.12130971, 0.16390255,
       0.12960353, 0.11012015, 0.12605913, 0.08225828, 0.12483724,
       0.07553666, 0.10880233, 0.16587763, 0.15890025, 0.17667218,
       0.12203955, 0.16270383, 0.13956173, 0.17609157, 0.1329435 ,
       0.17093259, 0.1487696 , 0.13642834, 0.12862892, 0.11986644,
       0.14018399, 0.13569863, 0.12352876, 0.1390964 , 0.13789728,
       0.11127125, 0.1740751 , 0.11412632, 0.15834849, 0.1430574 ,
       0.19045705, 0.14197344, 0.15991192, 0.1673336 , 0.08320697,
       0.11853421, 0.13975696, 0.10554477, 0.07266311, 0.1530208 ,
       0.14626922, 0.1599174 , 0.10922667, 0.15007749, 0.10078926,
       0.11527258, 0.09056383, 0.10290945, 0.11431269, 0.10214151,
       0.11281752, 0.14514863, 0.09825553, 0.12906459, 0.09839334,
       0.1310849 , 0.17151259, 0.08175332, 0.13591628, 0.17111905,
       0.07780149, 0.14896274, 0.15863752, 0.11145978, 0.16233292],
      dtype=float32)
                },
                'post': {
                    'b': array([ 0.00238558,  0.00539059,  0.00254566, ..., -0.00227536,
        0.00215697,  0.0025347 ], dtype=float32),
                    'w': array([[[-1.31261721e-02, -2.97298040e-02,  8.55484325e-03, ...,
          2.99458811e-03,  7.99451582e-03,  4.69723949e-03],
        [ 2.37402972e-02, -5.83681231e-03, -1.17273564e-02, ...,
         -4.31737164e-03, -1.61416568e-02,  5.95104136e-03],
        [-8.93504638e-03,  1.82599935e-03,  8.56268592e-03, ...,
         -1.26973065e-02, -5.25536062e-03, -6.47541368e-03],
        ...,
        [ 2.20728423e-02, -1.01757552e-02, -2.52463436e-03, ...,
         -8.62695649e-03,  1.56933945e-02, -1.77065507e-02],
        [ 4.22290387e-03, -1.19603090e-02,  8.87258165e-03, ...,
          2.84907804e-03, -1.67047736e-04,  3.99949402e-03],
        [ 6.23347703e-03,  2.00265143e-02, -9.35546495e-03, ...,
          1.93768144e-02, -1.65404938e-02,  5.31815866e-04]],

[[ 1.23761157e-02, -2.65776310e-02,  1.49613125e-02, ...,
          4.31999527e-02, -6.18449459e-03, -4.86998074e-03],
        [ 1.13966800e-02, -5.44342678e-03,  9.29086562e-03, ...,
         -1.73923075e-02, -2.29557697e-02, -8.23161285e-03],
        [ 5.82369696e-03, -5.08245826e-03,  1.27494999e-03, ...,
         -2.42798225e-04,  5.25014289e-03,  3.47217824e-03],
        ...,
        [ 2.28907121e-03, -1.06845573e-02,  7.44855218e-03, ...,
         -4.93692327e-03,  2.31771823e-03, -4.55397228e-03],
        [ 1.36834998e-02,  9.54026263e-03, -4.21440694e-03, ...,
          1.04895304e-03,  2.22359132e-03, -8.36855453e-03],
        [ 9.44504794e-03,  7.52265193e-03, -3.72815807e-03, ...,
         -9.25853476e-03, -3.18137021e-03, -6.19034225e-04]],

[[-1.78536344e-02,  3.85386660e-03, -5.28351869e-04, ...,
         -7.78203458e-03,  1.66550651e-02,  1.84652749e-02],
        [ 1.22876996e-02,  4.75762034e-04,  1.56713203e-02, ...,
          1.62448047e-03,  3.31607577e-03,  1.07453540e-02],
        [-2.35416763e-03,  2.51377723e-03,  1.56394839e-02, ...,
          4.00299998e-03, -9.81865171e-03, -2.82121735e-04],
        ...,
        [-1.86693370e-02, -1.13218045e-02, -1.17324227e-02, ...,
          1.66481305e-02,  5.23714954e-03,  1.63841657e-02],
        [ 7.23129138e-03,  2.89176661e-03,  2.89678457e-03, ...,
         -1.47060780e-02, -5.86029666e-04,  3.92893422e-03],
        [-4.33460763e-03, -2.94154603e-02,  5.65283699e-03, ...,
          9.21446644e-03, -2.06553470e-02, -1.20675052e-03]],

...,

[[-1.22889888e-03, -8.33128113e-03, -1.54948998e-02, ...,
         -9.16577410e-03, -3.76278115e-03, -1.07567674e-02],
        [-1.82199422e-02,  5.36414934e-03, -9.10198502e-03, ...,
         -2.56886636e-03,  5.26746549e-03,  2.85878079e-03],
        [ 8.39194166e-04, -1.52237006e-02,  2.40577610e-05, ...,
         -1.21503314e-02,  2.47731321e-02, -1.56330522e-02],
        ...,
        [-4.70445305e-03,  3.18478257e-03,  1.16151487e-02, ...,
         -7.27329915e-03,  7.05271633e-03, -9.84458346e-03],
        [ 3.46596260e-03,  1.12833101e-02,  1.45643270e-02, ...,
          1.03915539e-02, -2.00028978e-02, -6.42416812e-03],
        [ 1.76939380e-03, -1.86964884e-04,  2.39861733e-03, ...,
         -7.63287582e-03,  4.67931107e-03,  1.40486741e-02]],

[[ 9.50831734e-03,  5.77376317e-03, -4.75170062e-04, ...,
          3.36845331e-02, -2.35515404e-02,  1.57026220e-02],
        [-7.90429302e-03, -2.08003856e-02,  6.99472893e-03, ...,
         -1.18193971e-02, -6.12980872e-03, -5.21125412e-03],
        [-6.00326294e-03,  1.44797806e-02, -4.39825701e-03, ...,
          1.60151888e-02, -1.60791888e-03, -1.26069615e-04],
        ...,
        [ 2.60656769e-03, -2.01438293e-02, -5.43505093e-03, ...,
         -8.62396602e-03, -1.32938074e-02, -1.25113009e-02],
        [ 5.90699306e-03, -1.26863420e-02,  1.49284853e-02, ...,
         -4.33739260e-05, -3.53666884e-03,  8.76491517e-03],
        [ 4.19982756e-03,  6.60547987e-03, -9.10532661e-03, ...,
          1.36286225e-02,  6.15848345e-04,  6.93759043e-03]],

[[-1.76674947e-02, -3.76421376e-03,  1.47921732e-03, ...,
          1.29456343e-02,  4.27357294e-03,  5.15852123e-03],
        [ 2.25980859e-03,  1.39413550e-02,  7.51878368e-04, ...,
          4.16671112e-03,  2.33600629e-04, -4.58875642e-04],
        [ 2.03958916e-06, -2.20134063e-03, -8.97115748e-03, ...,
         -1.51937432e-03,  1.53056076e-02,  4.09533083e-03],
        ...,
        [-2.85116630e-03,  2.10857601e-03,  5.37850382e-03, ...,
         -3.66166560e-03, -1.59080066e-02,  4.00653947e-03],
        [ 1.76424384e-02, -8.02353863e-03,  1.91531260e-03, ...,
          1.64852082e-03,  2.21708702e-04,  2.77315639e-03],
        [-5.73996827e-03,  1.43148312e-02,  4.94344532e-03, ...,
         -1.78103801e-02, -2.68698321e-03, -1.45744148e-03]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.40734804, 0.41428447, 0.38511682, ..., 0.39734745,
         0.39927614, 0.37339577],
        [0.37582898, 0.3782926 , 0.35838625, ..., 0.36785197,
         0.37276763, 0.38693842],
        [0.40043324, 0.38810897, 0.35555586, ..., 0.3340775 ,
         0.36437902, 0.35953525],
        ...,
        [0.40284282, 0.3815503 , 0.37254843, ..., 0.37550068,
         0.39063406, 0.37736753],
        [0.34199855, 0.3192871 , 0.35416415, ..., 0.37947404,
         0.3294963 , 0.29134592],
        [0.35526   , 0.3903868 , 0.36237094, ..., 0.35171282,
         0.38711283, 0.33790264]]], dtype=float32)
                },
                'query': {
                    'b': array([[ 6.9429409e-03, -4.2634704e-03, -2.7791863e-02, ...,
        -1.0255985e-02, -1.0615559e-05, -2.6138142e-02],
       [-1.2106268e-03, -1.1853982e-02, -3.1291507e-02, ...,
         3.1470947e-02, -4.6250102e-04, -3.4001581e-02],
       [-5.2930247e-02, -2.2960834e-02, -2.2257559e-02, ...,
         5.8740806e-02, -3.5103314e-02,  4.9581606e-02],
       ...,
       [-6.5015107e-03,  1.1675995e-02,  8.1795076e-04, ...,
         1.6125415e-03,  5.1611493e-04,  1.4375899e-02],
       [ 9.2227589e-03, -9.7014355e-03,  1.6348656e-02, ...,
        -3.0453945e-02,  7.7625657e-03, -1.3052466e-02],
       [-2.1310246e-03, -1.3935447e-02, -7.7975774e-04, ...,
         1.2236374e-02, -2.9945837e-03, -3.2102251e-03]], dtype=float32),
                    'w': array([[[-3.16672074e-03, -4.12859917e-02,  3.21716592e-02, ...,
          2.45895912e-03,  8.08126386e-03, -9.22187325e-03],
        [-6.76687434e-03, -2.07112320e-02,  5.13712093e-02, ...,
          7.17079872e-03, -2.47327350e-02,  1.19166179e-02],
        [ 2.89679803e-02,  2.35539190e-02, -7.39458501e-02, ...,
          1.19256554e-02,  2.08058786e-02, -3.49808764e-03],
        ...,
        [ 2.40502618e-02, -2.07911748e-02,  8.93004518e-03, ...,
         -2.51299459e-02,  2.76682843e-02,  4.03891085e-03],
        [ 2.68634390e-02,  3.79768834e-02,  2.77351253e-02, ...,
         -2.39275396e-03, -4.56889421e-02, -2.58478243e-02],
        [ 1.85000955e-03, -1.42436158e-02, -9.64279659e-03, ...,
          1.45759247e-02,  7.56929256e-03, -2.44231988e-02]],

[[ 8.27130582e-03,  1.81877185e-02, -2.55307532e-03, ...,
         -1.52691249e-02, -3.25792353e-03, -2.96841521e-04],
        [ 4.34543453e-02,  5.64510282e-03,  1.77895054e-02, ...,
          2.63340049e-03, -2.14000698e-02, -1.05866799e-02],
        [-6.36522938e-03,  2.85282396e-02,  1.20690363e-02, ...,
         -6.00102218e-03,  1.37423193e-02,  8.13120045e-03],
        ...,
        [ 1.47966743e-02, -2.69153379e-02, -3.99247110e-02, ...,
          1.65948849e-02,  2.78216600e-02, -1.45663070e-02],
        [-1.61692854e-02,  1.30160823e-02,  1.63440332e-02, ...,
         -2.56844088e-02, -2.32111895e-03,  6.39781505e-02],
        [ 1.45603744e-02,  4.30756900e-03,  1.74908843e-02, ...,
          1.05988989e-02,  4.51415256e-02,  1.59411468e-02]],

[[ 3.04652918e-02, -1.33064361e-02,  7.95358326e-03, ...,
         -4.94426396e-03, -2.80136503e-02, -1.09789921e-02],
        [-7.48527609e-03,  8.50938261e-03,  1.11475568e-02, ...,
         -7.09244516e-04, -6.31795629e-05, -5.71189122e-03],
        [-2.20030714e-02,  2.95007718e-03,  1.46583728e-02, ...,
         -6.68075075e-03,  2.16055289e-03,  1.61396209e-02],
        ...,
        [-1.04833571e-02, -5.54391183e-03,  2.82914005e-02, ...,
          2.07205694e-02,  5.66454837e-03, -5.68461837e-03],
        [-1.17350044e-03, -9.73211695e-03,  1.57576025e-04, ...,
          1.04285050e-02, -2.52882205e-03, -8.29409715e-03],
        [ 2.47114687e-03,  5.86273940e-03,  7.58946128e-03, ...,
         -2.62915669e-03,  1.73396263e-02, -1.10322051e-02]],

...,

[[-1.37799522e-02,  1.82114318e-02, -3.75177450e-02, ...,
         -1.14078913e-02, -3.51569094e-02, -1.70358531e-02],
        [ 8.99701938e-03,  1.50934095e-02,  1.34488260e-02, ...,
          1.25771090e-02,  1.32271769e-02, -2.91112054e-04],
        [ 2.51405928e-02,  1.53764468e-02,  1.26926471e-02, ...,
         -3.65321524e-02,  1.28214108e-02, -2.43623834e-02],
        ...,
        [-2.75633018e-02,  8.98138341e-03,  3.26420926e-02, ...,
          4.59148549e-02, -5.57082742e-02, -1.65827069e-02],
        [-2.87620537e-02, -3.28576416e-02, -1.20256692e-02, ...,
          2.24592187e-03,  2.69954875e-02,  1.02574239e-02],
        [ 5.26716094e-03, -1.30683286e-02, -5.21436520e-03, ...,
         -1.56574827e-02,  1.66685320e-02, -4.16750694e-03]],

[[ 5.96964434e-02, -4.42794263e-02, -3.15744206e-02, ...,
         -2.08790954e-02,  1.49590028e-02, -1.30561497e-02],
        [-1.00302100e-02,  5.13463886e-03,  3.51827219e-02, ...,
         -1.93800870e-02, -1.62844229e-02,  3.18349041e-02],
        [ 3.20788138e-02,  6.39357269e-02, -4.81731817e-02, ...,
          6.96237013e-03,  4.43332829e-02, -3.86579446e-02],
        ...,
        [-2.53242403e-02, -1.82418469e-02,  1.63953025e-02, ...,
          3.15922312e-02, -3.47541273e-02, -4.47433442e-02],
        [ 2.01659314e-02, -2.34309863e-03,  3.89689207e-02, ...,
          4.04491462e-03,  1.96682215e-02, -3.30764391e-02],
        [ 8.85207392e-03,  1.24980835e-02, -1.48997540e-02, ...,
         -5.80404140e-03, -1.28095532e-02,  1.04766097e-02]],

[[-6.21598586e-03,  8.93101934e-03, -1.86101720e-02, ...,
          1.13250483e-02,  1.06790354e-02, -2.76543386e-02],
        [ 1.37867238e-02, -8.71839374e-03, -3.37770469e-02, ...,
         -1.38885714e-02, -1.04870023e-02, -1.42791821e-02],
        [-8.93813092e-03,  2.45561004e-02,  6.13623625e-03, ...,
          4.75202966e-03,  1.26813389e-02,  4.04227711e-03],
        ...,
        [ 2.35408582e-02, -8.72598030e-03, -1.76732186e-02, ...,
          2.81246589e-03,  2.43727863e-02, -1.42596839e-02],
        [-2.70808507e-02, -2.46383362e-02, -2.90598087e-02, ...,
         -6.43516891e-03,  3.99530344e-02,  3.38392779e-02],
        [-2.20168545e-03,  1.46329468e-02, -7.26030860e-03, ...,
         -2.56383326e-03, -5.55172889e-03,  1.80497952e-02]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.65668344, 0.6663939 , 0.7726092 , ..., 0.68539494,
         0.6354839 , 0.9234263 ],
        [0.66794515, 0.64477164, 0.5327006 , ..., 0.97633505,
         0.6281436 , 0.6034146 ],
        [0.4897735 , 0.55887765, 0.76579577, ..., 0.73630434,
         0.585252  , 0.52807426],
        ...,
        [0.78400433, 0.57725596, 0.7720839 , ..., 0.7868061 ,
         0.7222632 , 0.6482354 ],
        [0.53842926, 0.5179979 , 0.67649937, ..., 0.47779468,
         0.5332574 , 0.69582903],
        [0.5697829 , 0.5771688 , 0.45513922, ..., 0.6232267 ,
         0.61543465, 0.5839382 ]]], dtype=float32)
                },
                'value': {
                    'b': array([[-1.0620573e-03, -8.5262273e-04, -2.3737380e-03, ...,
         1.3075229e-03, -1.5832923e-03, -4.5124182e-04],
       [-1.0804366e-03,  8.5270044e-04,  2.5770220e-03, ...,
        -2.9048433e-03, -3.7585336e-04, -4.6495227e-03],
       [ 2.8058852e-03,  2.1279312e-03,  1.8310938e-03, ...,
         3.3984135e-03, -8.4306856e-05,  1.5666549e-03],
       ...,
       [ 1.6452165e-03,  3.6395458e-04, -8.0337049e-04, ...,
         2.2301511e-04,  3.2636215e-04, -7.0281944e-04],
       [ 6.3909398e-04,  1.7140268e-03, -7.0529274e-04, ...,
        -1.2073840e-03,  1.0770707e-03, -1.3171440e-03],
       [-6.2980293e-04, -1.3103519e-03,  3.3602221e-03, ...,
         3.4310108e-03, -3.3526905e-03,  3.5224902e-04]], dtype=float32),
                    'w': array([[[-0.00289195,  0.00459336, -0.00052058, ..., -0.02344152,
         -0.01150526, -0.00118089],
        [-0.02425805,  0.01403235,  0.0003381 , ..., -0.0124934 ,
          0.04205531, -0.02273614],
        [-0.00022628,  0.00789206, -0.01157164, ..., -0.00642179,
          0.00537121,  0.00377361],
        ...,
        [-0.0134025 , -0.02511169,  0.02299355, ..., -0.00510895,
         -0.0051118 ,  0.01501753],
        [ 0.01340156,  0.00708515, -0.00953312, ..., -0.00929322,
         -0.00020737, -0.00195217],
        [-0.00274176, -0.01731715,  0.01001492, ..., -0.00381206,
          0.03188181,  0.00717415]],

[[-0.01997502,  0.00705155,  0.00379616, ...,  0.00206456,
         -0.01684273, -0.01042701],
        [-0.01811747,  0.01008563, -0.01092734, ...,  0.00223534,
          0.01655077, -0.02580835],
        [ 0.02445975, -0.01113591,  0.00088073, ...,  0.00551457,
         -0.0136836 , -0.00647382],
        ...,
        [-0.00687634, -0.00110594, -0.0010924 , ..., -0.03426222,
         -0.0004331 ,  0.01047713],
        [-0.00492099, -0.00675182,  0.00610867, ...,  0.0057382 ,
          0.02763575, -0.0040886 ],
        [ 0.00776301,  0.02418736, -0.01555426, ...,  0.00088955,
          0.00354905, -0.00647146]],

[[ 0.00591547, -0.00494045, -0.00427117, ...,  0.01073097,
         -0.00779866,  0.00647531],
        [-0.02251022,  0.00660691,  0.0009929 , ...,  0.00659384,
         -0.00468486, -0.01036458],
        [ 0.01986051,  0.00580555, -0.00189009, ..., -0.00222066,
         -0.00057134, -0.00097204],
        ...,
        [ 0.00790048, -0.00821528,  0.00801988, ..., -0.01131336,
         -0.00314801, -0.02870887],
        [ 0.00733948, -0.00031473,  0.0028545 , ...,  0.0082247 ,
          0.0108723 ,  0.00233836],
        [ 0.01011771,  0.00825984, -0.00927334, ...,  0.00065992,
          0.00597515, -0.00673824]],

...,

[[-0.02173947,  0.00239984,  0.00772854, ...,  0.02920106,
          0.00906544,  0.00643171],
        [-0.00138991,  0.00526376,  0.00399699, ...,  0.00990278,
         -0.00578325,  0.00019137],
        [-0.01732081,  0.00318549,  0.00263875, ...,  0.00073776,
          0.01109647, -0.00948012],
        ...,
        [ 0.00383234,  0.01830041, -0.0235064 , ..., -0.00297496,
         -0.00463046,  0.01416991],
        [-0.01552369,  0.00129807, -0.02198475, ...,  0.01483279,
          0.01075404, -0.00188153],
        [-0.00974719, -0.00263777,  0.00704402, ...,  0.00187992,
         -0.00388174, -0.00453114]],

[[-0.00326155,  0.00924625, -0.00512391, ..., -0.03187041,
          0.01513776, -0.01164287],
        [ 0.00969861,  0.01691657,  0.00360392, ...,  0.00483298,
          0.02489362,  0.00192794],
        [-0.01393731, -0.00638645,  0.00695994, ..., -0.00465638,
          0.00981852, -0.0306015 ],
        ...,
        [-0.00763948,  0.00937049,  0.00952126, ..., -0.0129568 ,
         -0.00520916,  0.01216789],
        [ 0.01056911,  0.00709295, -0.00374232, ...,  0.01133963,
          0.00611105, -0.01102026],
        [-0.00244382,  0.00323424,  0.00117606, ..., -0.01558896,
         -0.02972729, -0.00575228]],

[[ 0.00171981,  0.0112906 , -0.02009648, ..., -0.00829016,
          0.00360181, -0.00637888],
        [ 0.01464567, -0.020429  , -0.0022388 , ...,  0.00657347,
         -0.01722888,  0.00407795],
        [-0.01575869, -0.00118728, -0.00671169, ...,  0.00680375,
          0.00524415,  0.01559517],
        ...,
        [-0.00369396, -0.00475997, -0.00338401, ..., -0.00040036,
          0.00523991, -0.00302144],
        [ 0.00544856, -0.00823792, -0.02051219, ...,  0.01267082,
         -0.01351453, -0.00835027],
        [ 0.00199931,  0.00934889,  0.00237274, ...,  0.00907233,
         -0.00732404, -0.00648139]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.4618968 , 0.48530924, 0.44078627, ..., 0.46062115,
         0.44134542, 0.42632702],
        [0.44628635, 0.4539724 , 0.42003033, ..., 0.42763895,
         0.45110437, 0.43132395],
        [0.44681305, 0.43616533, 0.42098743, ..., 0.37801364,
         0.4234676 , 0.39948902],
        ...,
        [0.4371776 , 0.43009245, 0.42688635, ..., 0.42733753,
         0.4557684 , 0.43100303],
        [0.4124495 , 0.35433814, 0.41060773, ..., 0.47520396,
         0.39858687, 0.33478233],
        [0.40375882, 0.44376042, 0.41227964, ..., 0.40754393,
         0.45541832, 0.36834267]]], dtype=float32)
                }
            }
        },
        'x_layers_7': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': array([-0.00571089,  0.00145912, -0.00137052, ..., -0.00824607,
       -0.00131992, -0.00115036], dtype=float32)
                    },
                    'linear': {
                        'w': array([[-0.02234673, -0.02884459,  0.0098833 , ..., -0.02191785,
        -0.00469763, -0.04039831],
       [-0.03696799, -0.04514505,  0.01396954, ..., -0.01213033,
        -0.00943626,  0.04773937],
       [ 0.01503859, -0.02884225,  0.02720422, ..., -0.02438783,
        -0.0169282 , -0.00341779],
       ...,
       [ 0.01055255, -0.00945783, -0.00152049, ..., -0.03797672,
         0.01715031,  0.02279662],
       [ 0.03525571,  0.00862676,  0.02131652, ...,  0.00314574,
        -0.00221583,  0.01686604],
       [ 0.02903666, -0.03785585,  0.0386798 , ..., -0.00419276,
        -0.00648742, -0.02684806]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[0.95695746, 1.1264538 , 0.9941023 , ..., 0.983062  , 1.0339077 ,
        1.0909111 ]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': array([ 0.0026283 , -0.00116334,  0.0034741 , ..., -0.00103938,
        0.00077989,  0.00055067], dtype=float32)
                    },
                    'linear': {
                        'w': array([[ 4.3947481e-02,  4.8813175e-02,  7.3814997e-03, ...,
         5.8599636e-03,  3.8389336e-02, -4.0372342e-02],
       [ 3.5060907e-03, -2.8329035e-02,  2.6754966e-02, ...,
         6.0271759e-02,  4.4384837e-02, -4.1161864e-03],
       [ 1.5060810e-02,  4.7639436e-03, -6.8287635e-03, ...,
         8.3255162e-03, -2.5526772e-03,  9.9604297e-03],
       ...,
       [-2.9504925e-02,  2.0276491e-02,  4.6000529e-02, ...,
        -1.9012368e-04,  3.0562451e-02,  4.3479633e-02],
       [ 2.4119338e-02, -8.0203616e-05, -1.5165000e-02, ...,
         1.1062592e-02,  2.3118423e-03, -7.7793556e-03],
       [ 3.0921111e-02,  3.3886950e-02,  4.4420842e-02, ...,
        -1.1219447e-02,  1.9637524e-04,  1.2746536e-02]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.6575729, 1.4231606, 1.9481348, ..., 1.6508765, 1.8229539,
        1.623804 ]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': array([ 0.00490241,  0.00695948,  0.00014633, ..., -0.00513277,
       -0.00615356,  0.00304212], dtype=float32),
                    'scale': array([-0.00079224,  0.00206569, -0.01471212, ...,  0.00290086,
        0.0069622 , -0.00268389], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': array([1.0908813, 1.1087157, 1.0528654, ..., 1.0645745, 1.0824969,
       1.0966488], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': array([[-2.3981717e-03,  1.8856628e-03, -5.0984562e-04, ...,
         7.3413196e-04,  2.2099728e-03, -6.8922009e-04],
       [-2.7342146e-04, -2.5460121e-04, -2.0622846e-03, ...,
        -1.6955294e-03, -6.1627716e-04,  6.4225256e-04],
       [-4.1961134e-03,  1.7308042e-04,  2.5284584e-04, ...,
         1.2709869e-03,  3.9050930e-05,  7.5965037e-04],
       ...,
       [ 5.3820893e-04,  2.3627002e-03, -8.7761186e-04, ...,
         2.3640343e-03, -9.1595820e-04,  2.5802132e-04],
       [ 1.3188351e-03, -1.7600934e-03, -2.8453977e-04, ...,
         3.7302570e-03, -1.5706382e-03, -1.0020508e-03],
       [-1.8914823e-03, -7.5478945e-04, -2.6961186e-03, ...,
        -4.9656915e-04, -1.6461979e-03,  3.2195733e-03]], dtype=float32),
                    'w': array([[[ 6.40782015e-03, -3.66740656e-04,  5.17848786e-03, ...,
          1.36679262e-02, -3.99049465e-03,  6.54312270e-03],
        [ 1.33482981e-02, -5.79426842e-06,  1.14805858e-04, ...,
          2.80219043e-04, -2.07124953e-03, -1.47480005e-02],
        [ 9.00816638e-03, -3.81786795e-03,  5.01242885e-03, ...,
         -7.67782563e-03, -4.90695750e-03, -9.86438617e-03],
        ...,
        [-2.30965801e-02, -2.56713610e-02, -2.57862173e-02, ...,
         -3.67835164e-04,  3.84535571e-03, -1.73336025e-02],
        [-1.53349172e-02, -1.23835681e-02, -1.58335790e-02, ...,
          1.68166608e-02, -1.33521706e-02, -8.03984515e-03],
        [ 9.64884926e-03,  2.03878060e-02,  1.40785463e-02, ...,
          4.21672575e-02,  2.65351832e-02, -9.15440917e-03]],

[[ 3.20037007e-02, -1.66236702e-02,  2.32836567e-02, ...,
          1.04394453e-02, -4.10447968e-03,  2.89984024e-03],
        [ 8.79825931e-03,  4.22275858e-03, -7.79476808e-03, ...,
         -1.07526742e-02, -1.84024416e-03,  1.18503990e-02],
        [-5.07890945e-03, -5.92159387e-03, -2.61693005e-03, ...,
          1.57626718e-03,  2.54532089e-04,  4.61818138e-03],
        ...,
        [ 5.79911517e-03,  4.92377952e-03,  2.18899734e-02, ...,
         -2.25461964e-02,  1.12297032e-02,  5.18489210e-03],
        [ 5.59712714e-03,  1.07457750e-02, -1.70159005e-02, ...,
          1.35056172e-02, -1.57417152e-02,  1.02928677e-03],
        [ 3.99082433e-03,  1.09748859e-02, -1.47347664e-02, ...,
          1.99719183e-02,  1.98588166e-02,  5.31212194e-03]],

[[ 9.96251963e-03, -4.61663585e-03, -9.71068814e-03, ...,
         -1.05131399e-02,  1.10108126e-03,  3.01431841e-03],
        [-1.81468180e-03,  7.67473551e-03, -9.29079857e-03, ...,
          3.79140303e-03, -9.37093107e-04,  5.18163154e-03],
        [-6.91172900e-03,  3.68022665e-05, -2.64968537e-03, ...,
         -1.53994923e-02,  9.56442486e-03,  1.13882590e-02],
        ...,
        [ 4.64544073e-03,  2.15418600e-02,  7.16026625e-05, ...,
         -1.09560797e-02,  6.27882546e-03, -3.17468122e-03],
        [ 1.46364821e-02,  2.29149312e-02, -2.52039842e-02, ...,
          4.18484397e-03,  1.01864822e-02, -2.46211328e-03],
        [-5.78414788e-03,  5.80046559e-03, -6.54958840e-03, ...,
         -8.30895826e-03, -1.40840863e-03, -4.89032036e-03]],

...,

[[ 9.54452902e-03, -7.88491312e-03,  1.89009476e-02, ...,
          1.79626215e-02, -2.11949460e-02,  1.95353590e-02],
        [ 3.03449784e-03,  5.76388929e-03,  4.99098888e-03, ...,
          6.98912248e-04, -1.22431805e-02,  4.67736460e-03],
        [-9.36352182e-03,  9.05931368e-03,  5.36272768e-03, ...,
          1.82745773e-02,  4.59127594e-03,  1.67307968e-03],
        ...,
        [ 1.91309415e-02, -2.55044666e-03,  9.86091327e-03, ...,
         -5.18217217e-03,  1.15610417e-02,  1.32541005e-02],
        [ 5.20884059e-03, -5.08935610e-03,  9.62142181e-03, ...,
         -7.90964346e-03,  1.02058914e-03,  8.71451385e-03],
        [-2.23930110e-03, -2.34254287e-03,  4.81208507e-03, ...,
          1.02508254e-02, -6.91749109e-03, -5.89943584e-03]],

[[ 1.46664644e-03,  5.07657183e-03,  7.24075828e-03, ...,
         -1.37436297e-02, -3.52300727e-03, -5.71473455e-03],
        [-8.14115722e-03,  2.02324285e-04, -8.40366539e-03, ...,
          7.77847366e-03, -2.29069758e-02,  1.05358846e-02],
        [-3.95153387e-04,  6.99168025e-03,  9.70429555e-03, ...,
          5.51025011e-03, -4.83118789e-03,  5.76234702e-03],
        ...,
        [-2.18965784e-02, -1.77675039e-02, -1.29633397e-02, ...,
         -1.42749341e-03, -1.87776366e-03, -2.03180406e-02],
        [ 2.54297443e-02, -7.58786872e-03,  3.07482691e-03, ...,
         -8.74846475e-04, -1.35524245e-02,  1.71892217e-03],
        [-1.52709195e-02, -1.00328447e-02,  1.76302216e-04, ...,
          4.70977189e-04, -3.46529251e-03,  4.57736431e-03]],

[[-1.41176418e-03,  1.28303468e-02,  1.05975266e-03, ...,
          1.96604617e-02,  2.04978511e-02,  2.20396370e-03],
        [-2.38677618e-04,  1.28345722e-02,  9.21138446e-04, ...,
          2.24298518e-02,  1.20315086e-02, -2.85336189e-02],
        [ 1.73810627e-02,  3.50343733e-04,  4.90271952e-03, ...,
         -1.35342497e-02, -2.17246432e-02, -2.02825442e-02],
        ...,
        [-9.97872464e-03, -3.68483714e-03,  1.12736169e-02, ...,
          8.30859691e-03,  4.30642674e-03, -5.83485328e-03],
        [-2.17700992e-02,  9.59218945e-03, -1.46980500e-02, ...,
          4.30577202e-03,  5.33139473e-03,  3.57369259e-02],
        [ 1.92339625e-03, -1.11738732e-02,  9.66002641e-04, ...,
         -6.37061149e-03, -3.23438871e-04,  6.01156382e-03]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.40430248, 0.4013894 , 0.47193074, ..., 0.490303  ,
         0.43466747, 0.46967933],
        [0.56253403, 0.55481994, 0.5412978 , ..., 0.449288  ,
         0.55757326, 0.51592463],
        [0.45119837, 0.4992445 , 0.41844743, ..., 0.45310876,
         0.41885322, 0.45596433],
        ...,
        [0.5322837 , 0.53946745, 0.49735332, ..., 0.50600123,
         0.41434696, 0.46288136],
        [0.49172598, 0.43603677, 0.50176954, ..., 0.42157704,
         0.49965635, 0.4779243 ],
        [0.560492  , 0.54345554, 0.48736274, ..., 0.5007944 ,
         0.42227405, 0.43580738]]], dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': array([0.1745563 , 0.17456126, 0.14752585, 0.15875179, 0.1832802 ,
       0.17058335, 0.13296197, 0.17121601, 0.10767658, 0.15312862,
       0.16568439, 0.1583921 , 0.1848713 , 0.1689837 , 0.16452795,
       0.16034178, 0.16015925, 0.18659249, 0.19397616, 0.16274922,
       0.1499158 , 0.16354154, 0.18811893, 0.16714635, 0.15575878,
       0.169097  , 0.17320454, 0.16842654, 0.18550313, 0.16002789,
       0.14593089, 0.18536483, 0.16758567, 0.12782294, 0.16751379,
       0.18410124, 0.13181742, 0.19628252, 0.18065505, 0.16620027,
       0.16837107, 0.17566574, 0.1590769 , 0.16995479, 0.15416765,
       0.17122333, 0.2122131 , 0.17859052, 0.18375066, 0.15738262,
       0.16987279, 0.1377341 , 0.14713049, 0.19109888, 0.16073835,
       0.155179  , 0.18291654, 0.15882567, 0.1332647 , 0.14301917,
       0.16009846, 0.13951255, 0.15479031, 0.17047878, 0.159181  ,
       0.12073389, 0.13865942, 0.16203314, 0.1777702 , 0.15428174,
       0.16125916, 0.1845427 , 0.1468957 , 0.17631026, 0.13524841,
       0.12261588, 0.12875122, 0.16690533, 0.16385828, 0.16424495],
      dtype=float32)
                },
                'post': {
                    'b': array([ 0.00290152,  0.00115534,  0.00654437, ..., -0.0028905 ,
        0.0003559 ,  0.00298951], dtype=float32),
                    'w': array([[[ 1.02901012e-02,  1.41829038e-02, -1.01600485e-02, ...,
          6.96739834e-03,  1.74732395e-02, -3.62850353e-03],
        [ 1.74780264e-02,  1.75960902e-02, -7.69804139e-03, ...,
         -1.47320535e-02, -1.33710250e-03,  3.97499353e-02],
        [-7.04897457e-06,  2.79059093e-02,  1.67463962e-02, ...,
          1.56734269e-02,  6.61494210e-04, -8.48551840e-03],
        ...,
        [ 1.02333287e-02,  3.67542845e-03, -1.01666525e-02, ...,
          1.26873637e-02,  8.48016702e-03,  1.77150797e-02],
        [ 8.24190117e-03,  5.60097629e-03,  1.18684843e-02, ...,
          2.06912514e-02,  2.93256715e-03, -7.79098505e-03],
        [-1.25470618e-02, -1.41636077e-02, -2.62559578e-03, ...,
         -7.30138179e-03,  4.54861205e-03, -1.10310428e-02]],

[[ 1.08056646e-02,  3.01855174e-03, -4.09245538e-03, ...,
         -1.67644303e-02, -4.17756476e-03,  1.59451496e-02],
        [ 7.00694928e-03, -1.21578472e-02, -1.66128539e-02, ...,
         -5.24787407e-04, -4.30331507e-04,  1.79922692e-02],
        [-5.33438404e-04,  1.64916907e-02,  5.75977331e-03, ...,
         -4.24635084e-03, -1.01308658e-04, -1.79685522e-02],
        ...,
        [ 1.07242856e-02,  1.24283441e-04, -2.08249409e-03, ...,
          2.48522926e-02, -4.92088124e-03,  1.14400946e-02],
        [ 4.53855284e-03,  4.15513897e-03, -1.86227094e-02, ...,
         -1.05627831e-02, -1.49715515e-02, -1.77796255e-03],
        [ 8.21541809e-03,  6.36926037e-04,  1.05452519e-02, ...,
          2.40012491e-03,  1.48209035e-02,  6.98841596e-03]],

[[ 2.16584317e-02,  5.55987284e-03,  1.48854153e-02, ...,
          1.03149116e-02, -4.74577257e-03,  1.44185219e-02],
        [ 4.73785866e-03,  1.42107131e-02,  1.37145668e-02, ...,
          7.00110197e-03,  7.64492573e-03, -2.89369579e-02],
        [-2.02995669e-02, -4.53712326e-03,  4.82046278e-03, ...,
         -5.30065224e-03,  4.61907871e-03,  9.27641930e-04],
        ...,
        [-5.20071853e-03, -1.50685189e-02, -1.16542298e-02, ...,
          7.89072248e-04,  1.32440189e-02, -1.18319606e-02],
        [ 1.72421914e-02, -4.51067416e-03, -1.63098257e-02, ...,
         -8.48147646e-03, -8.23695504e-04,  1.44770388e-02],
        [-1.11587215e-02, -7.10662734e-03, -3.93924816e-03, ...,
         -2.61096116e-02, -2.12753122e-03, -8.19018111e-03]],

...,

[[-8.85027181e-03,  9.26445238e-03,  7.49548152e-03, ...,
          1.45644788e-02, -1.11973165e-02,  1.63836591e-03],
        [-4.52435808e-03,  2.05388130e-03,  6.44828565e-03, ...,
         -6.85672229e-03, -5.30801713e-03, -2.31105592e-02],
        [ 1.77382901e-02,  2.40481386e-04, -4.08038357e-03, ...,
          1.95335690e-02,  8.14477727e-03, -4.12678253e-03],
        ...,
        [-1.47519610e-03,  1.03230141e-02,  1.37564288e-02, ...,
         -2.43476056e-03,  2.04248750e-03, -2.25722251e-04],
        [-1.09387925e-02,  4.40103235e-03,  1.56049486e-02, ...,
          7.04761967e-03, -9.88948718e-03, -1.52529785e-02],
        [-7.05215475e-03,  1.84222346e-03, -8.80805030e-03, ...,
          3.70892370e-03,  3.77923971e-05, -2.52227648e-03]],

[[-4.45318205e-04,  2.43835081e-03,  8.79021082e-03, ...,
         -1.63909104e-02,  5.53428475e-03, -2.56436318e-03],
        [-4.06683888e-03, -7.63842370e-03, -1.28814392e-02, ...,
          1.98927289e-03,  1.27030257e-02,  1.41252736e-02],
        [-2.66466872e-03,  5.08026825e-03,  7.91727845e-03, ...,
          1.51418038e-02, -2.05588397e-02,  1.79581176e-02],
        ...,
        [ 2.46931519e-02,  6.08667731e-03,  4.41043591e-03, ...,
         -2.97294394e-03, -1.93123193e-03, -8.25054944e-03],
        [-7.63998087e-03, -1.25631336e-02, -6.44728600e-04, ...,
          2.70021590e-03, -1.34934569e-02,  1.32549191e-02],
        [ 2.07663956e-03,  1.24648111e-02, -1.31936623e-02, ...,
          2.27751490e-02, -2.54688971e-03,  1.01049030e-02]],

[[-8.00714921e-03,  1.30047426e-02,  1.02177646e-03, ...,
         -4.53452085e-04, -1.41068490e-03, -4.64247027e-03],
        [ 8.46171845e-03,  7.21889921e-03, -1.17666358e-02, ...,
         -8.61446839e-03, -1.08559737e-02,  8.59782379e-03],
        [-4.55562910e-03,  3.41780542e-04, -3.09505779e-03, ...,
          1.27979526e-02, -9.39201564e-03,  1.39818527e-02],
        ...,
        [ 3.17281141e-04, -8.64622369e-03,  3.42501700e-03, ...,
          2.67034722e-03, -4.23402665e-03,  2.99277948e-03],
        [-1.84059367e-02, -4.27844189e-03,  8.56903195e-03, ...,
          2.37914166e-04, -9.01622791e-03,  1.79223046e-02],
        [-9.79379378e-03,  1.14735682e-02,  1.30470167e-03, ...,
         -3.09733092e-04, -1.67014338e-02,  3.92003683e-03]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.37763   , 0.40522468, 0.3819962 , ..., 0.3948339 ,
         0.3728904 , 0.38398725],
        [0.38455513, 0.3861339 , 0.38555682, ..., 0.37586272,
         0.37628815, 0.39047208],
        [0.38696465, 0.40023667, 0.37714985, ..., 0.40954152,
         0.39641857, 0.40642697],
        ...,
        [0.39897984, 0.4285953 , 0.37198237, ..., 0.3870524 ,
         0.39261866, 0.37394822],
        [0.37640315, 0.3510961 , 0.3898991 , ..., 0.3964075 ,
         0.37818858, 0.37709635],
        [0.36172706, 0.34663314, 0.39248854, ..., 0.38945457,
         0.3908186 , 0.36470908]]], dtype=float32)
                },
                'query': {
                    'b': array([[ 0.01523319, -0.00828504, -0.00129177, ...,  0.00156883,
        -0.00982505, -0.00408466],
       [-0.00088482, -0.00184363,  0.00965276, ...,  0.00431157,
        -0.00666857, -0.00053838],
       [-0.01299675,  0.00013154,  0.02363259, ..., -0.0216933 ,
         0.00750832,  0.01218794],
       ...,
       [-0.00368615,  0.00494643,  0.00254787, ..., -0.00205449,
         0.01362027,  0.02188244],
       [ 0.01115793, -0.00408845,  0.00487758, ...,  0.00150502,
        -0.00290738, -0.0086401 ],
       [ 0.00153861, -0.00713248, -0.00014092, ..., -0.00658943,
        -0.01316522, -0.00693501]], dtype=float32),
                    'w': array([[[-1.13905529e-02, -3.67759634e-03, -2.04983577e-02, ...,
          1.11687044e-02,  5.42940281e-04,  1.82931740e-02],
        [ 1.40451295e-02, -9.68987588e-03,  2.07039956e-02, ...,
          3.94661725e-02,  1.94060840e-02,  1.49192326e-02],
        [-7.23185530e-03,  1.94731783e-02,  3.10550374e-03, ...,
         -2.07566041e-02,  3.11420206e-03,  7.74376141e-03],
        ...,
        [-1.05216971e-03, -2.06391374e-03, -3.61125767e-02, ...,
         -2.35695615e-02, -1.78337395e-02,  1.17942831e-02],
        [ 9.29886755e-03,  2.45120320e-02,  7.42621627e-03, ...,
          1.73580907e-02, -7.13619683e-03, -2.02737600e-02],
        [ 6.38788752e-03,  1.45135447e-02,  5.60244266e-03, ...,
         -1.47173805e-02, -6.62107253e-03,  1.58582255e-02]],

[[ 3.42304981e-03, -7.53982458e-03, -1.60115659e-02, ...,
         -2.70423032e-02, -1.88085344e-02, -4.41422639e-03],
        [-1.90075557e-03, -1.86322350e-02, -9.16051865e-03, ...,
          2.98879799e-02,  1.11626377e-02,  3.17776091e-02],
        [-2.06668582e-02,  3.85817178e-02,  2.71899812e-02, ...,
         -1.17414491e-02,  1.88219436e-02,  4.91011105e-02],
        ...,
        [ 3.13950293e-02,  5.69591373e-02,  6.02741865e-03, ...,
          2.97739934e-02,  6.79989252e-03, -1.65666863e-02],
        [-1.25809079e-02, -4.43746615e-03,  2.43168734e-02, ...,
          1.50622975e-03, -1.31668700e-02, -1.44779533e-02],
        [ 4.93501686e-03, -1.02419006e-02,  1.12571595e-02, ...,
         -1.93927512e-02,  2.18026023e-02,  1.35928066e-02]],

[[ 1.17914367e-03,  4.78289463e-03, -1.77173559e-02, ...,
         -3.24882865e-02, -5.45305712e-03, -6.62063155e-03],
        [ 5.44125354e-03,  7.38598546e-03, -5.67020476e-03, ...,
          4.00381498e-02, -1.31989652e-02,  7.77168432e-04],
        [-1.34885013e-02,  8.46647471e-03, -1.04275197e-02, ...,
         -3.00852656e-02,  9.50143076e-05,  1.06741181e-02],
        ...,
        [ 2.27596276e-02,  3.94316390e-02,  1.02557922e-02, ...,
          7.03268684e-03,  2.01479550e-02, -2.36197095e-02],
        [-3.46829481e-02, -4.75858757e-03,  3.65067087e-02, ...,
          4.39764336e-02, -2.33494509e-02,  5.97192207e-03],
        [-3.89813930e-02,  1.43961264e-02, -7.16604944e-03, ...,
          1.16381301e-02, -2.41715256e-02, -4.68559563e-03]],

...,

[[ 8.96142330e-03, -9.32922494e-03, -2.57806722e-02, ...,
         -2.91875694e-02, -9.69469827e-03,  9.88694374e-03],
        [-4.72861007e-02,  3.99811529e-02, -2.77204439e-02, ...,
          3.08614224e-02, -9.48628224e-03, -9.32027120e-03],
        [-1.17529053e-02,  2.19715107e-02,  2.84017138e-02, ...,
         -6.02748394e-02,  3.74339670e-02,  6.22763373e-02],
        ...,
        [ 3.54887508e-02,  8.27263203e-03,  3.61161865e-02, ...,
         -3.71246561e-02,  5.24913222e-02,  1.07056592e-02],
        [-1.43796746e-02, -6.49276655e-03,  8.91252328e-03, ...,
         -8.98791850e-03, -1.04523879e-02, -5.00745419e-03],
        [-6.58036023e-03, -2.48913281e-02,  6.72871654e-04, ...,
          1.72955971e-02, -8.02072580e-04,  1.60879467e-03]],

[[ 1.47671509e-03, -2.34128255e-02, -6.71872171e-03, ...,
          1.46008085e-03,  3.04142386e-03, -3.96831334e-03],
        [-6.96668169e-03, -1.26852449e-02,  2.60265241e-03, ...,
          9.51035786e-03, -1.93178449e-02, -2.00089943e-02],
        [ 6.41523395e-03,  2.65257526e-02, -2.45728507e-03, ...,
         -6.26919605e-03, -3.26306163e-03, -5.66462521e-03],
        ...,
        [ 6.32555410e-02,  1.64253935e-02, -4.24433406e-03, ...,
         -1.26359137e-02, -7.37594161e-03,  3.69609054e-03],
        [ 2.37846486e-02, -1.96743850e-03, -4.88712359e-03, ...,
          7.45793013e-03,  1.76147406e-03,  1.53834582e-03],
        [ 1.86960176e-02,  9.77923255e-03, -7.88097456e-03, ...,
          5.24946814e-03,  2.11336892e-02,  2.70112394e-03]],

[[-1.50356777e-02,  5.00446605e-03, -1.51625159e-03, ...,
          4.74293064e-03,  2.27355901e-02, -6.27850043e-03],
        [ 4.54321653e-02, -2.74610631e-02, -4.70345244e-02, ...,
          8.63380730e-03,  4.13595140e-02,  2.94508738e-03],
        [-1.30670536e-02, -2.23860908e-02,  4.23285691e-03, ...,
         -3.14619625e-04, -4.14622389e-02,  3.56040038e-02],
        ...,
        [-5.40730469e-02, -1.84541214e-02,  2.73042731e-02, ...,
         -2.20951382e-02,  2.24010423e-02, -1.97337382e-02],
        [ 7.33314129e-03,  1.78060699e-02, -2.36569867e-02, ...,
         -8.27676803e-03,  2.98645347e-02, -1.18271064e-03],
        [-1.29001159e-02,  1.34417852e-02,  2.15053628e-03, ...,
         -2.29821657e-03,  2.24603470e-02, -6.90755760e-03]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.53726876, 0.5004498 , 0.5274452 , ..., 0.55016077,
         0.6155445 , 0.5155556 ],
        [0.612324  , 0.6087871 , 0.76558846, ..., 0.7650917 ,
         0.6641285 , 0.6407085 ],
        [0.65969986, 0.68597686, 0.61169374, ..., 0.61563283,
         0.6862773 , 0.74138236],
        ...,
        [0.83999836, 0.6725453 , 0.7911588 , ..., 0.714955  ,
         0.7237057 , 0.86569506],
        [0.6051847 , 0.50430614, 0.6335796 , ..., 0.5442495 ,
         0.59184915, 0.6184985 ],
        [0.5865929 , 0.65711254, 0.6780168 , ..., 0.68165827,
         0.58227915, 0.5378038 ]]], dtype=float32)
                },
                'value': {
                    'b': array([[ 1.5917098e-03,  1.4896401e-03, -2.0226375e-03, ...,
        -1.7325892e-03,  2.9298672e-03, -2.7322539e-03],
       [-3.1144351e-03, -4.1114488e-03,  1.0012481e-03, ...,
         4.8245918e-03,  6.4365431e-03, -1.2060608e-03],
       [ 4.5222994e-03,  9.4918645e-04, -6.6103740e-04, ...,
         1.8348355e-03,  3.3785610e-03, -7.6923724e-03],
       ...,
       [-7.8128959e-04,  1.6019873e-03,  8.1217266e-04, ...,
        -4.9950148e-04, -2.2263450e-03, -1.9558065e-03],
       [-1.9730320e-03, -1.4390795e-03,  1.7571831e-03, ...,
         3.8916760e-03, -2.0562236e-03, -1.1587031e-03],
       [-3.9481013e-03,  3.2905554e-03,  4.4394241e-05, ...,
         2.0804864e-03, -5.7885433e-03,  3.5198012e-03]], dtype=float32),
                    'w': array([[[-1.60310082e-02, -9.71163157e-03, -1.20064905e-02, ...,
         -1.51673928e-02, -8.96296173e-04, -6.39644917e-03],
        [-1.70423854e-02, -6.60409639e-03,  7.88445026e-03, ...,
          2.02861018e-02, -2.18733568e-02, -2.00053845e-02],
        [ 4.50440589e-03, -8.13141465e-03,  1.07267508e-04, ...,
          7.52942683e-03,  1.85728399e-03,  9.87506006e-03],
        ...,
        [-1.12143280e-02, -3.10050487e-03,  1.23486603e-02, ...,
         -4.45978865e-02, -7.39333080e-03, -6.61169877e-03],
        [-1.48295267e-02, -5.04418463e-03, -1.81792956e-02, ...,
         -1.82359703e-02, -6.24101004e-03,  6.44446258e-03],
        [ 1.23566040e-03, -5.22012264e-03,  3.13698314e-03, ...,
          1.32481931e-02,  6.28043199e-03, -4.75604553e-03]],

[[-9.46191791e-03, -2.74957884e-02,  1.44271851e-02, ...,
         -8.95215571e-03, -1.34227220e-02, -1.55676743e-02],
        [-8.02108739e-03, -1.83729362e-03, -5.57104219e-03, ...,
          8.38356279e-03, -7.60852173e-03, -3.92554002e-03],
        [ 3.65592539e-03,  2.51928181e-03,  2.52209930e-03, ...,
          1.86118726e-02, -8.83613247e-03,  1.20169679e-02],
        ...,
        [-1.20420419e-02, -2.86604986e-02,  2.22302582e-02, ...,
         -4.94761541e-02, -2.90343096e-03, -1.65365767e-02],
        [-1.88420229e-02, -2.03898782e-03,  4.14783368e-03, ...,
         -4.75509046e-03, -5.22224838e-03, -7.94941373e-03],
        [ 1.77196600e-02, -1.39220180e-02, -5.72877703e-04, ...,
         -6.17535179e-03, -3.04477266e-03,  4.59991675e-03]],

[[-3.97885498e-03,  2.36060075e-03, -2.47439067e-03, ...,
          1.02601293e-02, -5.00805955e-03, -3.02892085e-02],
        [-4.61665494e-03, -2.18351111e-02, -9.82285198e-03, ...,
          3.23216361e-03, -5.42462012e-03,  1.59116499e-02],
        [ 3.55587554e-06,  1.05092386e-02,  5.12951845e-03, ...,
          3.56717128e-03, -6.35621091e-03, -1.24632055e-02],
        ...,
        [-1.02671217e-02,  1.38035640e-02,  1.19161941e-02, ...,
         -4.33072494e-03, -6.77717268e-04,  6.87065476e-04],
        [-2.96325758e-02, -6.37733983e-03, -1.41574442e-03, ...,
         -3.04651633e-02,  8.08874727e-04,  1.70397162e-02],
        [-1.16946420e-03,  4.16479073e-03, -1.00080669e-03, ...,
          5.73403388e-03, -1.90013200e-02,  4.16753674e-03]],

...,

[[-2.99818814e-03,  1.03562130e-02, -1.25704519e-02, ...,
          5.97063918e-04,  9.91802756e-03,  1.58828534e-02],
        [ 1.14250637e-03,  8.65176693e-03,  8.48595984e-03, ...,
         -1.19797243e-02,  6.41112914e-04,  2.01795418e-02],
        [-1.84944253e-02, -3.53778945e-03, -7.62337027e-03, ...,
          1.59696694e-02, -2.69793696e-03,  7.62890140e-03],
        ...,
        [-1.00193582e-02, -1.15135051e-02,  1.44954515e-03, ...,
          3.03447945e-03,  6.84753945e-03, -6.31584320e-04],
        [ 7.57189235e-03, -3.23931947e-02, -9.54238605e-03, ...,
          1.11592608e-02, -6.86746323e-03,  1.55623397e-02],
        [-4.57788259e-03, -8.10335949e-03, -3.30413366e-03, ...,
          6.53423183e-03, -3.00777378e-03, -6.36744592e-03]],

[[-1.31105119e-02, -1.34802451e-02,  1.71116169e-03, ...,
         -1.20635061e-02,  5.15042385e-03, -1.60549860e-02],
        [ 3.85765987e-03,  1.29135596e-02,  2.92267427e-02, ...,
         -2.02956181e-02, -9.77604557e-03, -1.62428524e-02],
        [ 1.44871036e-02, -6.62202947e-03,  2.52859350e-02, ...,
         -1.64018758e-03,  1.07800886e-02, -7.46442704e-03],
        ...,
        [-1.88547503e-02, -1.54293096e-02,  2.16310704e-03, ...,
          1.53392013e-02,  2.15644203e-03,  1.02116037e-02],
        [ 1.21050631e-03, -1.25914644e-02, -2.55856812e-02, ...,
         -4.69026938e-02,  1.24884322e-02,  3.29828672e-02],
        [-7.00702332e-03,  4.62156441e-03, -1.72035526e-02, ...,
          4.35407786e-03,  8.94513074e-03,  7.08126416e-03]],

[[-1.17207430e-02, -1.41995978e-02, -1.24308169e-02, ...,
         -1.09773008e-02, -6.52138842e-03, -4.35515959e-03],
        [ 2.22678925e-03,  7.40520703e-03, -1.84388403e-02, ...,
          4.69964603e-03,  2.42842291e-03, -2.51398217e-02],
        [ 3.94491130e-04,  1.39072891e-02,  1.91890746e-02, ...,
         -5.70005318e-03,  2.28017895e-03, -9.48434789e-03],
        ...,
        [-2.89604533e-03,  1.19845793e-02, -9.42718890e-03, ...,
          1.52631383e-02, -5.83796343e-03, -1.43370975e-03],
        [ 9.60272551e-03, -1.21000046e-02,  8.05047061e-03, ...,
         -1.12463962e-02,  9.17629153e-03, -8.07019940e-04],
        [-3.64702418e-02, -1.07744811e-02,  1.80233661e-02, ...,
         -1.90189574e-02,  1.28748957e-02, -7.23076938e-03]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.44865337, 0.4704645 , 0.43833014, ..., 0.44745937,
         0.43675956, 0.45570928],
        [0.4216957 , 0.44772282, 0.44491592, ..., 0.4187714 ,
         0.42973554, 0.46098796],
        [0.44933164, 0.4759536 , 0.4571385 , ..., 0.46125278,
         0.46058673, 0.46048236],
        ...,
        [0.4664991 , 0.48288023, 0.4311082 , ..., 0.44994286,
         0.445075  , 0.40820748],
        [0.43695226, 0.42430666, 0.46252316, ..., 0.45483184,
         0.433405  , 0.45308587],
        [0.4133735 , 0.42686078, 0.48992193, ..., 0.45205677,
         0.45824438, 0.43511665]]], dtype=float32)
                }
            }
        },
        'x_layers_8': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': array([-0.00671606, -0.00744732, -0.01157051, ..., -0.00302582,
       -0.00751212, -0.00731062], dtype=float32)
                    },
                    'linear': {
                        'w': array([[-0.01811136, -0.04249045, -0.02398836, ...,  0.03573981,
        -0.06078571, -0.00195698],
       [ 0.04663501, -0.01439486,  0.00278891, ...,  0.01190364,
         0.03039053, -0.02769112],
       [-0.02636963,  0.03524577,  0.02645216, ..., -0.04963003,
         0.02481861, -0.01761094],
       ...,
       [ 0.03440456, -0.05879504, -0.0126651 , ...,  0.05787901,
        -0.03594774, -0.0137208 ],
       [ 0.00762873, -0.01527641, -0.00911149, ..., -0.00852254,
         0.01500743, -0.00605547],
       [ 0.0276429 ,  0.00140306,  0.02717231, ...,  0.00757653,
        -0.05295376, -0.0348709 ]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.0048705, 1.0787035, 1.0907592, ..., 1.1749966, 1.1022477,
        1.0469747]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': array([ 1.7641198e-03,  7.2908530e-05,  2.7227844e-03, ...,
       -7.4495800e-04, -3.3278845e-04,  2.1731786e-03], dtype=float32)
                    },
                    'linear': {
                        'w': array([[-0.03619846,  0.01551942,  0.0302618 , ...,  0.00047247,
        -0.01799299,  0.04210168],
       [ 0.03249831, -0.0137839 ,  0.04283076, ...,  0.01192124,
         0.01020726,  0.01523312],
       [-0.0461726 , -0.00367335,  0.03881538, ...,  0.03333255,
         0.05790588,  0.00927986],
       ...,
       [ 0.00713237,  0.00093277,  0.00547557, ..., -0.02959165,
        -0.01509296, -0.03668424],
       [ 0.0416071 , -0.01149079,  0.06314063, ...,  0.05279715,
        -0.02821371, -0.02391061],
       [-0.03470379, -0.03203568,  0.01379244, ..., -0.01838429,
         0.03363138,  0.02721118]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.5409344, 1.4072804, 1.842395 , ..., 1.5180601, 1.7194685,
        1.7001759]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': array([ 0.00412433,  0.00395123,  0.00951787, ..., -0.00058707,
       -0.00273577,  0.00690942], dtype=float32),
                    'scale': array([-0.0213481 ,  0.00577652,  0.00104609, ..., -0.00437833,
        0.00163347,  0.01052476], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': array([1.1052154, 1.1062342, 1.0680627, ..., 1.0658808, 1.0893704,
       1.1054778], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': array([[ 2.0952977e-03,  1.5934280e-03,  1.4717099e-03, ...,
        -1.6025685e-03, -1.7999718e-03, -3.2979084e-04],
       [-2.6650492e-03,  1.3669929e-03,  3.8853692e-04, ...,
         4.1401391e-03, -8.2425791e-04, -1.0697824e-03],
       [-1.2259793e-03, -2.6739226e-03, -5.6987611e-04, ...,
        -7.4962771e-04,  1.7805287e-03,  8.2909531e-04],
       ...,
       [-7.6656679e-06,  9.1410882e-04, -2.2895208e-03, ...,
        -1.3394810e-03, -4.1470411e-03, -9.1867056e-04],
       [-1.6828876e-03,  7.3032081e-04, -3.9617313e-04, ...,
        -7.6685828e-06, -1.3311633e-03, -8.0681883e-04],
       [ 5.7779968e-04,  1.3798820e-04,  1.4514620e-03, ...,
        -9.1995212e-04,  1.8731453e-05, -9.3555247e-04]], dtype=float32),
                    'w': array([[[-0.00633128,  0.00398494,  0.01548496, ..., -0.00509497,
          0.00089846,  0.00314374],
        [-0.00360716, -0.00279797, -0.03103199, ..., -0.0191979 ,
          0.00687667, -0.00903005],
        [ 0.00590024,  0.00502111, -0.00143865, ...,  0.00133199,
          0.0051042 , -0.01144267],
        ...,
        [-0.0215076 , -0.01586042, -0.02274709, ...,  0.01589907,
          0.01593778, -0.0125594 ],
        [ 0.0002792 , -0.00019325, -0.02181054, ..., -0.00606402,
          0.00556961, -0.00287801],
        [ 0.01454523, -0.0023863 , -0.00274182, ...,  0.00334038,
         -0.00116689,  0.00762898]],

[[-0.01151426, -0.00844785, -0.02125369, ...,  0.00603621,
         -0.00136562, -0.00762851],
        [ 0.01286156, -0.00166835,  0.00660577, ...,  0.01049994,
          0.00463221,  0.0228478 ],
        [ 0.02115211,  0.00557891,  0.00027872, ...,  0.0067573 ,
         -0.00773142, -0.02113993],
        ...,
        [-0.00284054,  0.00682781,  0.00642822, ..., -0.00628127,
         -0.00430453, -0.00668168],
        [-0.01064933, -0.01177286,  0.00995388, ..., -0.00817136,
          0.00586369,  0.03024234],
        [ 0.00196051, -0.00225924,  0.01046471, ...,  0.00295889,
          0.00273268,  0.01158652]],

[[-0.00427446, -0.00199157, -0.01781184, ..., -0.00617924,
          0.00506486, -0.00896436],
        [ 0.01785816, -0.02341854,  0.00280491, ..., -0.02231157,
         -0.00457481,  0.01940677],
        [ 0.00377306,  0.01018646,  0.01017737, ...,  0.00291785,
          0.00332073, -0.01921549],
        ...,
        [-0.00164351,  0.00988911,  0.03455843, ..., -0.01058343,
          0.01021736, -0.00178535],
        [ 0.02088065,  0.0137523 , -0.0152242 , ...,  0.01122727,
          0.0299897 , -0.04980445],
        [-0.00665353, -0.01346631,  0.00541798, ..., -0.03650415,
          0.00307006,  0.01412219]],

...,

[[ 0.0029633 , -0.0190199 , -0.00145295, ..., -0.01025373,
         -0.01129338, -0.01982854],
        [ 0.00068125,  0.01374491,  0.00305722, ...,  0.00659582,
          0.02870743,  0.01441414],
        [ 0.01073766, -0.01473211,  0.01719271, ...,  0.03191563,
          0.01529151,  0.01080893],
        ...,
        [ 0.00825522,  0.02644498,  0.02134844, ..., -0.00397444,
          0.00477742, -0.00307009],
        [ 0.00321409,  0.00963919,  0.00427253, ..., -0.01651971,
          0.00714024, -0.0040468 ],
        [ 0.00882799, -0.00824147,  0.00371395, ...,  0.00951616,
          0.00724876,  0.00576993]],

[[-0.02614701, -0.00488679, -0.0220326 , ...,  0.03407804,
          0.02856459, -0.03861711],
        [-0.01579829, -0.01094657,  0.02753543, ..., -0.00036637,
         -0.00937735, -0.00209734],
        [-0.00794617,  0.00011716, -0.00553119, ...,  0.00793986,
         -0.00225238,  0.00663604],
        ...,
        [-0.01192919, -0.02378488, -0.00405361, ...,  0.01092807,
         -0.0059597 , -0.01402603],
        [-0.01772652,  0.01845129,  0.00490151, ...,  0.00319138,
         -0.01406406, -0.00651068],
        [ 0.00398742, -0.00027455,  0.00097198, ..., -0.00815327,
         -0.00315835,  0.00252889]],

[[-0.0001781 , -0.01084789,  0.00013601, ..., -0.00540317,
         -0.00850195, -0.02196166],
        [ 0.01238866,  0.00744494, -0.01020275, ...,  0.00031833,
          0.01465734,  0.01575067],
        [-0.00223196, -0.00378659, -0.00145925, ..., -0.00451101,
          0.00256217, -0.00203178],
        ...,
        [ 0.02033761,  0.00679272, -0.00348961, ...,  0.00539042,
          0.01451758,  0.00278541],
        [ 0.00109349,  0.00856665, -0.00203083, ...,  0.01030129,
          0.01096501,  0.00195871],
        [ 0.01040758, -0.01207248,  0.02535666, ...,  0.01254702,
          0.01569652, -0.01007556]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.41806298, 0.5161992 , 0.43867406, ..., 0.5060122 ,
         0.4491196 , 0.5146279 ],
        [0.42865407, 0.4422233 , 0.46115795, ..., 0.56179196,
         0.5450001 , 0.44522578],
        [0.4471406 , 0.5005573 , 0.552917  , ..., 0.6047088 ,
         0.5242858 , 0.500268  ],
        ...,
        [0.45421934, 0.51572436, 0.48554403, ..., 0.44519746,
         0.41361693, 0.4400282 ],
        [0.5069249 , 0.48067835, 0.50832266, ..., 0.4521985 ,
         0.48456293, 0.51789826],
        [0.39661252, 0.46773842, 0.4798412 , ..., 0.49480534,
         0.44235438, 0.44522834]]], dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': array([0.16659452, 0.2176413 , 0.20913084, 0.16684686, 0.14747609,
       0.13816825, 0.17060304, 0.1732533 , 0.18752515, 0.18463098,
       0.17490867, 0.17087072, 0.20064996, 0.12334649, 0.20583531,
       0.20181543, 0.14958781, 0.14743958, 0.14016852, 0.13809851,
       0.14118049, 0.13237341, 0.15868132, 0.13771881, 0.16029447,
       0.14321326, 0.15923475, 0.17038305, 0.14117105, 0.16902876,
       0.14414205, 0.0990934 , 0.14415134, 0.17895143, 0.16005166,
       0.1644718 , 0.15972514, 0.16798589, 0.16425624, 0.17587021,
       0.15190102, 0.18830405, 0.18460017, 0.13043234, 0.2084825 ,
       0.15443006, 0.1792815 , 0.21106526, 0.16622984, 0.15344281,
       0.17172691, 0.18035972, 0.15648267, 0.19878417, 0.14236848,
       0.19366716, 0.15613955, 0.12001559, 0.1338472 , 0.20615734,
       0.14170356, 0.18917897, 0.14692408, 0.13729933, 0.2014039 ,
       0.1762525 , 0.17112777, 0.19586515, 0.14007089, 0.15948854,
       0.11711452, 0.14105257, 0.1645794 , 0.12344601, 0.1573255 ,
       0.17944992, 0.12884043, 0.15109557, 0.12746932, 0.19371282],
      dtype=float32)
                },
                'post': {
                    'b': array([ 0.00145101,  0.00251218,  0.00792553, ..., -0.00019417,
       -0.00048331,  0.00213577], dtype=float32),
                    'w': array([[[ 1.27353165e-02, -3.06770834e-03, -5.33052022e-03, ...,
         -4.73721186e-03,  1.25906803e-02, -1.32778967e-02],
        [-1.10966377e-02, -1.76666062e-02,  4.20981180e-03, ...,
          9.97211505e-03,  6.93971943e-03,  6.30564895e-03],
        [ 4.62995982e-03, -2.66032148e-04,  2.08375137e-03, ...,
         -9.29566100e-03,  8.34042300e-03, -6.12970488e-03],
        ...,
        [ 5.17173670e-03,  4.32177912e-03, -1.01235826e-02, ...,
          1.53043903e-02,  1.35354849e-03,  9.87807103e-03],
        [ 6.90858811e-03,  1.98646523e-02, -2.52215681e-03, ...,
          7.05157081e-03, -1.67272799e-02, -3.48131009e-03],
        [ 1.39710959e-02, -1.07559208e-02, -2.13850965e-03, ...,
          9.27119167e-04, -3.73881147e-03,  8.89662828e-04]],

[[ 4.19707620e-04,  1.22087505e-02, -4.32405807e-03, ...,
         -5.75176766e-03,  1.63327400e-02, -4.08618478e-03],
        [ 5.25874645e-03, -4.98850690e-03,  1.38791641e-02, ...,
         -3.78630729e-03,  1.91278243e-03, -5.66291390e-04],
        [-3.29660475e-02,  2.48253322e-03, -1.79171376e-02, ...,
         -1.60007402e-02,  1.29292430e-02,  2.26214398e-02],
        ...,
        [-9.62404534e-03, -2.25293320e-02, -3.56850587e-03, ...,
         -1.04150400e-02, -6.60828548e-03,  1.88778397e-02],
        [-7.96976686e-03, -3.71670630e-03, -6.15493907e-03, ...,
         -1.78773815e-04,  1.39813023e-02,  2.46558860e-02],
        [-1.45430332e-02, -1.27258757e-02, -2.72967480e-02, ...,
         -9.97739751e-03,  1.01359421e-02,  1.37746297e-02]],

[[-1.93058641e-03,  4.03225422e-03, -1.57831199e-02, ...,
          1.84100805e-04,  4.42346645e-05, -2.39188410e-03],
        [-2.14369763e-02, -9.92416777e-03,  6.01445185e-03, ...,
         -2.23175529e-02,  9.20303911e-03, -1.21452846e-02],
        [-1.09605221e-02, -1.17221726e-02,  4.28027660e-03, ...,
          7.69669004e-03,  8.34415108e-03, -9.59581509e-03],
        ...,
        [ 1.58961620e-02,  3.37239471e-03,  5.04925242e-03, ...,
         -1.00660548e-02, -6.28015446e-03, -7.08675070e-04],
        [ 1.10407558e-03, -8.21546931e-03,  3.71202966e-03, ...,
         -5.91203803e-03,  9.15352162e-03, -9.45499819e-03],
        [-3.44982604e-03, -1.24192890e-02,  5.01128100e-03, ...,
         -2.44303979e-03,  8.74656718e-03, -6.54851133e-03]],

...,

[[-6.69577299e-03, -5.59542794e-03, -7.23796722e-04, ...,
          2.22098343e-02, -1.57444982e-03, -4.55162302e-03],
        [-1.20174407e-03,  1.10858670e-02, -2.60060616e-02, ...,
         -5.67125948e-03, -3.06202727e-03,  6.07461370e-05],
        [-1.11202514e-02, -1.90331638e-02, -2.13182159e-03, ...,
          1.22346841e-02,  1.27581730e-02, -4.90240986e-03],
        ...,
        [-4.00927523e-03, -2.65220786e-03, -1.38245737e-02, ...,
          3.80044221e-04, -1.66562144e-02,  1.89675018e-02],
        [-7.92303029e-03, -8.85255821e-03,  1.72557328e-02, ...,
         -9.50201228e-03, -2.14053169e-02, -1.20024681e-02],
        [-5.87277673e-03, -2.24461127e-02, -1.60243940e-02, ...,
         -8.93757585e-03,  1.04615921e-02,  3.20985308e-03]],

[[ 7.75599619e-03, -4.29206993e-03,  3.14271189e-02, ...,
          1.21478690e-02, -7.82030169e-03, -7.95452762e-03],
        [ 1.00533366e-02,  1.67573299e-02, -1.90856755e-02, ...,
          9.58601758e-03,  1.61651671e-02,  2.07473035e-03],
        [-6.21894287e-05,  1.32275745e-02,  1.84501559e-02, ...,
          7.51547690e-04, -2.01896299e-03, -1.18409917e-02],
        ...,
        [-1.32502476e-03,  2.83706235e-04,  1.66715588e-03, ...,
          1.55944712e-02, -9.83850472e-03,  1.75674651e-02],
        [-3.36146262e-03, -5.25929919e-03, -1.57528017e-02, ...,
         -2.26945174e-03,  1.33316712e-02, -1.83225982e-03],
        [ 1.26659973e-02, -5.38648153e-03, -5.68929641e-03, ...,
          2.07488034e-02, -2.92814197e-03,  1.45897828e-03]],

[[ 9.04358551e-03,  1.56450942e-02, -3.21527361e-03, ...,
          3.07923579e-03,  2.93071754e-03, -8.14707112e-03],
        [-5.97499078e-03, -1.44700357e-03, -3.24818585e-03, ...,
          1.24646854e-02,  7.60318292e-03, -1.23952399e-03],
        [-1.11762225e-03, -3.22844251e-03, -9.97110456e-03, ...,
          1.02657862e-02, -8.27095658e-03,  1.30888466e-02],
        ...,
        [ 6.26176689e-03,  2.16218941e-02, -5.57345664e-03, ...,
         -2.45052413e-03,  2.03148648e-03,  5.85137022e-05],
        [-8.74185003e-04,  2.17746552e-02, -1.16565423e-02, ...,
         -1.39208622e-02, -3.95378703e-03,  1.49394246e-02],
        [-1.21912425e-02, -2.50114570e-03, -8.13443586e-03, ...,
         -1.52346827e-02, -2.04536365e-03,  4.48474661e-03]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.40836242, 0.39917153, 0.40937313, ..., 0.41189927,
         0.39684224, 0.4171755 ],
        [0.39411208, 0.3825706 , 0.38247663, ..., 0.39677957,
         0.38526747, 0.36886784],
        [0.40545547, 0.39973176, 0.39840013, ..., 0.41635612,
         0.3949777 , 0.40610787],
        ...,
        [0.3637275 , 0.37967142, 0.36971682, ..., 0.3762412 ,
         0.35166192, 0.3969567 ],
        [0.37593523, 0.4130378 , 0.42156887, ..., 0.38357714,
         0.40333882, 0.40931198],
        [0.39595196, 0.39537105, 0.38262403, ..., 0.4043555 ,
         0.3919776 , 0.40687546]]], dtype=float32)
                },
                'query': {
                    'b': array([[ 2.4137442e-04,  5.9842686e-03,  2.4649238e-02, ...,
        -1.8480065e-03, -1.1731535e-02,  1.5507266e-03],
       [-2.3323870e-03, -2.1887743e-03,  1.6048562e-03, ...,
        -1.4903965e-02, -1.6681623e-02,  1.1353353e-03],
       [ 2.3728411e-03,  7.1505019e-03,  5.7950812e-05, ...,
         4.6056965e-03, -8.6643379e-03,  6.6809235e-03],
       ...,
       [-1.4967571e-02, -9.4790878e-03, -1.0470150e-02, ...,
         1.3970692e-02,  3.4786370e-03, -3.5595410e-03],
       [ 8.3640805e-03,  1.7302636e-02,  7.8781396e-03, ...,
         1.6122483e-02,  9.5468238e-03, -2.2607304e-03],
       [-2.2019206e-02,  3.1206079e-03,  6.5784846e-03, ...,
        -1.1271449e-02, -1.5769789e-02,  1.5105515e-02]], dtype=float32),
                    'w': array([[[-1.83182843e-02,  1.23743725e-03,  2.92542875e-02, ...,
         -3.55846807e-02, -9.71184485e-03,  1.23744260e-03],
        [ 3.53889056e-02, -2.28063986e-02, -2.86072167e-03, ...,
         -1.58963613e-02, -1.60532327e-05, -8.02235119e-03],
        [-1.85477771e-02, -4.17265072e-02, -1.83387920e-02, ...,
         -4.50981036e-03, -1.07226791e-02, -2.05805842e-02],
        ...,
        [-1.31660281e-02, -1.52303940e-02, -1.63004815e-03, ...,
          5.14348224e-03,  9.24873538e-03, -2.81867501e-03],
        [-3.78815550e-03, -5.74611276e-02, -1.33208502e-02, ...,
          3.44731589e-03, -2.08448209e-02, -1.28401117e-02],
        [ 7.96766113e-03,  1.09075643e-02, -8.05056933e-03, ...,
         -1.16875744e-03, -1.54173728e-02,  3.37275192e-02]],

[[ 9.25359037e-03,  4.86588478e-02,  2.39025354e-02, ...,
         -1.11687910e-02, -9.90096200e-03, -3.19324099e-02],
        [ 4.89404052e-03, -2.49847453e-02,  8.74039996e-03, ...,
         -8.14085454e-03, -4.52398090e-03,  2.71312110e-02],
        [ 5.68006514e-03, -7.95466453e-03,  8.94954894e-03, ...,
          1.95573121e-02, -1.00370366e-02, -1.16543612e-02],
        ...,
        [ 2.13883836e-02, -1.20033827e-02,  1.82265006e-02, ...,
         -1.48929283e-02, -3.15566622e-02, -1.82190333e-02],
        [-2.85373675e-03,  3.12556773e-02,  1.42031594e-03, ...,
          2.88261138e-02, -4.11030799e-02, -2.39800988e-03],
        [-3.89888696e-02, -5.95205743e-03, -1.23618850e-02, ...,
         -1.57547165e-02,  1.11313453e-02,  2.57914923e-02]],

[[ 4.37831357e-02,  1.76211577e-02,  3.24980319e-02, ...,
         -2.78523061e-02, -2.85221003e-02,  1.87001023e-02],
        [-6.55018073e-03,  3.85813578e-03, -7.17168441e-04, ...,
         -2.56292038e-02, -2.02884097e-02, -4.47966531e-03],
        [ 2.71965866e-03,  8.28019530e-03,  1.27890753e-02, ...,
         -7.55679561e-03, -8.85517895e-03, -7.09129684e-03],
        ...,
        [ 1.09562941e-03,  8.18615593e-03, -2.74075009e-03, ...,
          1.90628125e-04, -3.29986028e-03,  2.16446654e-03],
        [ 5.18469363e-02,  1.02471439e-02, -6.99939579e-03, ...,
          1.43273752e-02,  2.51579434e-02, -1.36294859e-02],
        [-3.84955704e-02, -1.95996882e-03,  9.75196064e-03, ...,
          3.77283781e-03, -6.13540178e-03,  1.03475871e-02]],

...,

[[ 2.51479223e-02,  3.59233730e-02, -1.45830428e-02, ...,
         -2.62193475e-02, -1.94747169e-02,  9.52162500e-03],
        [ 3.39633180e-03,  2.14464571e-02,  4.32241056e-03, ...,
          9.75516252e-03,  1.34280836e-02,  7.95514695e-03],
        [ 5.34502510e-03,  1.37361260e-02,  7.86298513e-03, ...,
          5.25865564e-03,  1.10866688e-02, -2.20224168e-02],
        ...,
        [ 1.01387659e-02,  9.03078541e-03,  5.07410290e-03, ...,
         -1.00649903e-02,  1.06083238e-02,  1.43983448e-02],
        [-9.86261759e-03,  2.37580240e-02,  1.65260513e-03, ...,
         -3.33003770e-03,  1.74283963e-02, -4.26193848e-02],
        [-7.43464101e-03,  4.00063116e-03,  5.82877323e-02, ...,
         -1.29690003e-02,  2.29875799e-02,  2.36781724e-02]],

[[-1.05117755e-02,  1.94241386e-02,  5.88363744e-02, ...,
          3.16638686e-02,  2.97669955e-02,  7.47378403e-03],
        [ 4.07938380e-03,  2.76588160e-03,  1.45817436e-02, ...,
          3.86199057e-02,  3.34549695e-02,  2.81807482e-02],
        [ 2.40493286e-03,  3.21862963e-03, -2.93535553e-03, ...,
          1.73776522e-02, -6.58960082e-03, -3.29069085e-02],
        ...,
        [-1.54531505e-02, -1.97425615e-02,  2.70724669e-02, ...,
          4.37688362e-03,  1.90750863e-02,  2.57884897e-02],
        [ 2.29432043e-02,  1.08313821e-02,  1.27129992e-02, ...,
          5.25886454e-02,  3.08530796e-02,  1.63603667e-02],
        [-2.29478744e-03,  2.02213246e-02, -9.74382833e-03, ...,
          6.31026039e-03, -1.47316782e-02,  1.10781919e-02]],

[[ 1.67931709e-02, -3.89110134e-03, -7.07024755e-03, ...,
         -1.75038241e-02, -1.41046401e-02, -1.22451773e-02],
        [-2.25986191e-03,  2.28019506e-02, -1.65104698e-02, ...,
          1.21603906e-02,  7.39587331e-03, -1.51593311e-04],
        [ 2.36524432e-03,  8.21779016e-03,  1.16037857e-02, ...,
          2.04978175e-02, -8.24823976e-03, -8.35203566e-03],
        ...,
        [-1.06447320e-02, -5.12641557e-02, -4.48293984e-02, ...,
          3.25738192e-02,  3.45098414e-02, -1.49965817e-02],
        [ 2.10662968e-02, -3.54700051e-02, -2.07886472e-02, ...,
         -1.63861848e-02,  4.22230270e-03,  7.40205422e-02],
        [ 2.63525583e-02,  2.26179305e-02,  2.27909572e-02, ...,
          4.20264248e-03, -9.29573365e-03,  6.99693430e-03]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.81551665, 0.82560295, 0.82444006, ..., 0.7951489 ,
         0.82820064, 0.69267106],
        [0.5422625 , 0.6026522 , 0.6126264 , ..., 0.5247561 ,
         0.5280767 , 0.59558165],
        [0.6128581 , 0.84550524, 0.55252635, ..., 0.58981997,
         0.5833048 , 0.84139377],
        ...,
        [0.6335962 , 0.5333983 , 0.7261733 , ..., 0.5879487 ,
         0.5971033 , 0.5784912 ],
        [0.7854952 , 0.8713373 , 0.8783836 , ..., 0.7851523 ,
         0.60140616, 0.7919486 ],
        [0.56622964, 0.6045963 , 0.5934587 , ..., 0.7131344 ,
         0.6862472 , 0.6669689 ]]], dtype=float32)
                },
                'value': {
                    'b': array([[ 4.1770306e-03,  5.1761670e-03, -6.7285066e-03, ...,
         2.4542394e-03, -2.6138593e-03, -7.2140631e-04],
       [-7.5517694e-04, -3.9448631e-03, -2.1563931e-03, ...,
        -3.8056246e-03,  5.6819087e-03, -4.0685167e-03],
       [-3.4288532e-04, -4.0260023e-03, -5.0083274e-04, ...,
        -7.6099916e-04, -1.9242569e-03, -5.2494492e-04],
       ...,
       [-2.9573161e-03,  1.5531457e-03, -1.6761255e-03, ...,
        -2.6534407e-03,  3.9581796e-03,  9.5842370e-05],
       [ 3.3524251e-03,  6.2615072e-05,  2.0030276e-03, ...,
         2.8712992e-03, -2.8626940e-03, -2.0652276e-03],
       [ 3.6921471e-04,  3.5650653e-03,  9.7389205e-04, ...,
         9.9158031e-04, -1.2612826e-03, -8.0347230e-04]], dtype=float32),
                    'w': array([[[-1.03718126e-02,  3.82453911e-02, -6.29646285e-03, ...,
          1.13670034e-02, -2.14453563e-02,  3.31411976e-03],
        [ 6.25608023e-03,  1.31760472e-02, -1.72357615e-02, ...,
         -2.18084734e-02,  4.69026039e-04,  1.76579040e-02],
        [-4.03056853e-03, -2.83134095e-02,  6.33138046e-03, ...,
          2.66450122e-02,  1.12062646e-02, -1.31124966e-02],
        ...,
        [-1.07738366e-02, -3.03360820e-03, -8.02534260e-03, ...,
         -1.35927927e-02, -7.88540300e-03,  2.46340707e-02],
        [-2.51693465e-02, -1.15373321e-02, -1.81824174e-02, ...,
          9.18297283e-03,  2.27255132e-02, -9.70875286e-03],
        [ 3.92003581e-02,  1.23058343e-02, -1.22627122e-02, ...,
         -3.73543636e-03, -1.97548363e-02, -3.19940806e-03]],

[[-1.02686491e-02, -9.68178734e-03, -2.91011017e-02, ...,
          1.39174901e-03, -1.83624923e-02,  7.99911749e-03],
        [-1.58109528e-04,  5.82498778e-03, -8.74073710e-03, ...,
         -1.67920440e-02,  6.06165873e-03, -3.96588026e-03],
        [ 3.58209526e-03, -1.91275068e-02, -3.22908163e-03, ...,
         -2.14443891e-03,  6.58416655e-03, -7.37907086e-03],
        ...,
        [-7.20687851e-04,  1.03550423e-02,  2.30090730e-02, ...,
          1.02106612e-02, -6.54760050e-03,  8.41254951e-05],
        [-1.26744332e-02, -6.41940488e-03, -1.51580209e-02, ...,
          6.58541801e-04,  2.38918923e-02, -6.78239251e-03],
        [ 3.96397291e-03,  1.38633754e-02,  6.41268399e-03, ...,
         -8.31568614e-03, -2.06491482e-02, -1.97862182e-02]],

[[ 2.13458724e-02,  3.17214569e-03, -9.06134676e-03, ...,
         -1.31701045e-02,  5.92758367e-03,  3.55292833e-03],
        [ 1.05144633e-02, -9.50844679e-03, -4.83982702e-04, ...,
         -1.92573154e-03,  6.42648013e-03,  1.11290552e-02],
        [-7.78299663e-03, -9.38099157e-03,  9.27382056e-03, ...,
          1.34440139e-03, -1.12291276e-02,  6.42540539e-03],
        ...,
        [ 1.27245877e-02,  7.63334660e-03,  2.18216004e-03, ...,
          1.49134919e-02,  6.53501775e-04, -3.62303271e-03],
        [ 9.02205892e-03,  2.39321101e-03, -1.85427256e-02, ...,
          1.25457463e-03,  2.09131930e-02, -1.93097778e-02],
        [-6.06138539e-03, -8.28116108e-03, -9.44155455e-03, ...,
          1.13414591e-02,  2.26092106e-03, -3.20590995e-02]],

...,

[[ 1.27339941e-02,  1.23900436e-02,  5.33356564e-04, ...,
         -1.31577738e-02, -1.41499355e-03,  7.65222497e-03],
        [ 1.44222323e-02, -5.33361686e-04, -4.67160577e-03, ...,
          1.36647718e-02,  4.50214976e-03,  6.34316867e-03],
        [ 2.03849357e-02, -1.73012726e-04, -5.47730131e-03, ...,
          7.93927535e-03, -1.46660749e-02, -7.99891632e-03],
        ...,
        [-6.73626829e-03, -1.20107038e-02, -3.50847957e-03, ...,
          1.34057933e-02, -1.59207918e-03,  8.17972142e-03],
        [-1.20544555e-02,  5.55198546e-03, -2.01862957e-02, ...,
         -1.56280082e-02, -9.21647344e-03, -5.99176204e-03],
        [ 3.09784501e-03,  5.09416033e-03,  1.33862995e-04, ...,
          1.04025723e-02, -1.11993542e-02,  1.47905769e-02]],

[[ 1.05775101e-02,  9.13704652e-03, -9.56058130e-03, ...,
         -1.70453284e-02,  1.19881909e-02,  1.05333999e-02],
        [ 3.02929319e-02, -2.28990987e-02,  1.42122703e-02, ...,
         -7.00119929e-03,  1.81458481e-02,  4.76381835e-03],
        [ 1.87043454e-02,  1.63048767e-02, -5.74660441e-03, ...,
          2.73633022e-02, -1.82441771e-02, -1.54173682e-02],
        ...,
        [-2.08543930e-02, -2.56956834e-02,  6.71728398e-04, ...,
          1.82189187e-03,  1.60320662e-02, -1.91105399e-02],
        [ 4.53941710e-03, -1.05978979e-03,  2.91732103e-02, ...,
         -2.22397083e-03, -5.70082292e-03,  1.03596766e-02],
        [-5.26233250e-03,  1.93588380e-02, -1.46204457e-02, ...,
         -2.23432016e-02,  5.36559615e-03,  9.88106430e-03]],

[[ 4.40953998e-03,  5.04396996e-03,  1.78099032e-02, ...,
          5.27709723e-03, -4.73994808e-03, -9.23941657e-03],
        [ 8.66723433e-03,  1.35799265e-02, -4.54121828e-03, ...,
         -3.35079362e-03, -1.27472188e-02,  1.04994085e-02],
        [ 4.40144475e-04, -3.29202670e-03, -1.68872643e-02, ...,
         -3.02552562e-02, -3.17824702e-03,  6.67277956e-03],
        ...,
        [-1.29628535e-02, -2.33154222e-02,  1.43350307e-02, ...,
         -1.01879351e-02, -1.74827613e-02, -4.70098993e-03],
        [ 6.43897522e-03,  3.64626106e-03,  2.52360757e-03, ...,
          1.22200791e-02,  4.17962018e-03, -5.20875631e-03],
        [ 6.19279826e-03,  1.06170094e-02, -9.20233130e-03, ...,
         -8.31399206e-03, -1.35130780e-02, -9.58402082e-03]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.4454746 , 0.4320375 , 0.43816176, ..., 0.44363707,
         0.435695  , 0.4693681 ],
        [0.4621361 , 0.41382068, 0.44283706, ..., 0.453511  ,
         0.43179816, 0.41828167],
        [0.47211108, 0.44415155, 0.46590912, ..., 0.48889756,
         0.47865105, 0.4856387 ],
        ...,
        [0.41532844, 0.43950364, 0.43132916, ..., 0.41959107,
         0.39849746, 0.4706367 ],
        [0.43308064, 0.46372727, 0.4854496 , ..., 0.43244925,
         0.48471352, 0.4717685 ],
        [0.4739961 , 0.46684864, 0.4595355 , ..., 0.46811172,
         0.47249165, 0.48011625]]], dtype=float32)
                }
            }
        },
        'x_layers_9': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': array([-0.00398949,  0.00553328, -0.00854538, ..., -0.01065977,
       -0.01085383,  0.10406972], dtype=float32)
                    },
                    'linear': {
                        'w': array([[ 0.00385022,  0.06122993,  0.02721928, ..., -0.03239259,
         0.01958293, -0.09112478],
       [-0.02464781,  0.01411509,  0.04660855, ...,  0.02175984,
        -0.00267561,  0.02156737],
       [-0.01745151, -0.017796  ,  0.00316033, ..., -0.01512817,
        -0.03007369, -0.04826004],
       ...,
       [ 0.03673194,  0.00620448,  0.00960318, ...,  0.00170189,
         0.03893861,  0.05012967],
       [-0.03062727,  0.01344013,  0.02024613, ..., -0.00557454,
         0.03241096,  0.02492862],
       [ 0.00941322, -0.03152687,  0.01848996, ..., -0.00659775,
         0.04427981, -0.02963124]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.0556995, 0.9502708, 1.0434346, ..., 1.2208543, 1.0806576,
        1.8669682]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': array([ 0.00206702,  0.00195731,  0.00262794, ...,  0.00019651,
       -0.00277163,  0.00101381], dtype=float32)
                    },
                    'linear': {
                        'w': array([[ 1.3016735e-02, -1.3118952e-02,  1.3808155e-02, ...,
        -1.3593927e-02,  1.9679509e-02,  2.5111713e-02],
       [ 1.8518310e-02, -1.8335573e-02,  5.6840559e-03, ...,
        -1.7456638e-02,  3.6089368e-02,  4.5100108e-02],
       [ 8.2706856e-03,  6.8996758e-03, -8.6885365e-03, ...,
        -5.9069094e-04,  2.7520629e-03, -4.7562275e-02],
       ...,
       [-7.4123950e-03, -1.9565022e-02, -1.4083363e-03, ...,
         5.1535361e-02, -2.9559454e-02, -2.9456580e-02],
       [ 2.9814113e-02,  9.0444470e-03,  6.7100316e-02, ...,
        -4.8956252e-03,  2.9108735e-02, -9.5660544e-05],
       [-1.1483442e-01,  7.0538826e-02, -1.7947572e-01, ...,
        -6.6035509e-02,  1.1661811e-01, -6.2948383e-02]], dtype=float32),
                        'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                        'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                        'dora_m': Array([[1.2591344, 1.3849407, 1.325359 , ..., 1.2145925, 1.3025405,
        1.4788513]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': array([ 0.00483031, -0.00146662,  0.00633642, ..., -0.00554054,
       -0.00470097, -0.0005318 ], dtype=float32),
                    'scale': array([-0.00955576,  0.00869239,  0.00359508, ..., -0.0093105 ,
        0.00919607,  0.01382339], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': array([1.1348704, 1.1547747, 1.1233397, ..., 1.0905623, 1.0844481,
       1.1467605], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': array([[-0.0038408 ,  0.00185112, -0.00173835, ...,  0.00336161,
         0.00030514,  0.00082349],
       [ 0.00067013, -0.00466094,  0.0019564 , ..., -0.00179259,
        -0.00069546,  0.00018717],
       [ 0.00160742,  0.00054143, -0.00041573, ...,  0.00107178,
         0.00017538,  0.00085705],
       ...,
       [ 0.00180693,  0.0006135 ,  0.00234484, ...,  0.00308815,
        -0.00226742, -0.00088748],
       [-0.00088424,  0.00179963, -0.00305551, ..., -0.00693885,
         0.00103404,  0.0019422 ],
       [-0.0019314 , -0.00187426,  0.00245309, ..., -0.00048181,
        -0.00064726, -0.00287785]], dtype=float32),
                    'w': array([[[ 0.012124  ,  0.00197903,  0.01721361, ..., -0.01422858,
         -0.00852192,  0.00429994],
        [-0.00209467,  0.0140839 , -0.02325453, ..., -0.00315203,
         -0.01315125,  0.02884499],
        [ 0.00509635, -0.03448816,  0.01469674, ...,  0.00184527,
          0.00365957, -0.01060968],
        ...,
        [ 0.00186667, -0.00185121, -0.00026951, ..., -0.00374656,
          0.01521753,  0.00336414],
        [-0.02210866, -0.00564335, -0.00732728, ..., -0.01133978,
          0.00138814, -0.00057869],
        [-0.0173609 , -0.03433024,  0.02274247, ..., -0.00359569,
         -0.01011684, -0.01354941]],

[[ 0.00034996, -0.00902286, -0.02474663, ...,  0.01778983,
         -0.00886714,  0.01499094],
        [ 0.00750206, -0.00745654,  0.01249904, ...,  0.00638959,
          0.00364278,  0.00862715],
        [-0.0063558 , -0.01219261, -0.01038099, ..., -0.03010534,
          0.0159702 , -0.0001074 ],
        ...,
        [ 0.00254835, -0.01605504,  0.0013674 , ..., -0.00767998,
         -0.00951638, -0.02310466],
        [-0.01815142, -0.00292693, -0.00214419, ..., -0.0093016 ,
         -0.00684318, -0.02416206],
        [-0.00584364,  0.03907287, -0.00384107, ...,  0.02054882,
         -0.00678092, -0.00136573]],

[[-0.01215438,  0.00240089,  0.001869  , ...,  0.00966227,
          0.01818509, -0.00645014],
        [-0.02090402, -0.01400763,  0.00799644, ..., -0.01602943,
          0.03376702,  0.00270221],
        [-0.00673315, -0.01569688, -0.00939439, ..., -0.00854048,
          0.0049209 ,  0.00061114],
        ...,
        [-0.00227049, -0.0069703 ,  0.01904852, ...,  0.01627467,
         -0.02020339, -0.01298738],
        [-0.0473048 , -0.020413  ,  0.01489721, ...,  0.00058377,
         -0.00360308,  0.0006294 ],
        [-0.00122461, -0.02431608,  0.00584911, ..., -0.01942164,
          0.0062611 , -0.01271128]],

...,

[[-0.00338148,  0.01715936,  0.00992662, ...,  0.00584938,
         -0.01790283,  0.02103392],
        [ 0.0107375 , -0.00624809,  0.01304185, ..., -0.00113904,
         -0.01936487, -0.01655799],
        [ 0.02046724,  0.02197216, -0.00820215, ..., -0.00472429,
         -0.02915956,  0.01226464],
        ...,
        [ 0.01988025,  0.00235497,  0.00859264, ..., -0.00409299,
         -0.00119526,  0.0035002 ],
        [-0.00969689, -0.00706153,  0.00334076, ..., -0.00651932,
          0.01437093, -0.00038618],
        [-0.00103866, -0.0083807 ,  0.00825352, ...,  0.02021397,
         -0.0008669 , -0.00393646]],

[[ 0.02119676, -0.00568261,  0.00920534, ..., -0.01194344,
         -0.01837671, -0.03558515],
        [-0.01285816, -0.00867935,  0.01133768, ...,  0.01106733,
          0.01622685, -0.01178937],
        [ 0.01073986, -0.02089515, -0.01296987, ..., -0.00571519,
         -0.00250481,  0.00481565],
        ...,
        [-0.00032635,  0.0116058 , -0.0077116 , ..., -0.00829475,
          0.00364742,  0.00182706],
        [-0.01040423, -0.00693676, -0.00199236, ..., -0.00306173,
          0.00904297,  0.00618701],
        [ 0.01287991,  0.024333  , -0.01161683, ...,  0.01969703,
         -0.00236721,  0.01943644]],

[[ 0.0008972 ,  0.00157054,  0.0047218 , ..., -0.01568508,
          0.01023996,  0.01188735],
        [-0.01286612, -0.02371632, -0.00060593, ...,  0.01111791,
          0.00755575, -0.02098973],
        [-0.02322051, -0.04816492, -0.00661085, ..., -0.00524904,
         -0.00385692, -0.01056321],
        ...,
        [ 0.00430305, -0.00331239,  0.00269268, ..., -0.00651208,
         -0.01047816,  0.00297151],
        [ 0.01365148,  0.01276014, -0.00482784, ..., -0.0023671 ,
         -0.01253744, -0.01830389],
        [ 0.00691162, -0.0166491 ,  0.00581269, ...,  0.02547496,
         -0.01401114, -0.00315702]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.5445873 , 0.5533701 , 0.51243824, ..., 0.49041024,
         0.52118725, 0.4962067 ],
        [0.54295546, 0.4757374 , 0.528842  , ..., 0.4963227 ,
         0.5912916 , 0.5780971 ],
        [0.44947153, 0.76934665, 0.5588749 , ..., 0.52884054,
         0.59893084, 0.4558163 ],
        ...,
        [0.39040655, 0.3447304 , 0.4743087 , ..., 0.4221102 ,
         0.42641377, 0.58564925],
        [0.5624394 , 0.4417551 , 0.44326144, ..., 0.42787212,
         0.41589466, 0.5675745 ],
        [0.45961046, 0.612482  , 0.4457871 , ..., 0.5026795 ,
         0.45533592, 0.48010504]]], dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': array([0.238903  , 0.22793801, 0.23768473, 0.2100576 , 0.23713684,
       0.27073467, 0.26104686, 0.2744409 , 0.23066047, 0.24973206,
       0.26910165, 0.22207716, 0.25862432, 0.23024742, 0.22372228,
       0.23422919, 0.24650474, 0.2856742 , 0.2793826 , 0.26312664,
       0.23928094, 0.26544127, 0.25757164, 0.2025107 , 0.20041946,
       0.28672475, 0.25480896, 0.22854814, 0.24869782, 0.23096187,
       0.2813165 , 0.27034017, 0.2569795 , 0.21727042, 0.28055146,
       0.27281886, 0.25968137, 0.27898005, 0.22259375, 0.2377495 ,
       0.22099793, 0.2507727 , 0.24659538, 0.22101505, 0.22632588,
       0.25414217, 0.24685644, 0.23780593, 0.25982112, 0.25838885,
       0.22764745, 0.24200216, 0.18154444, 0.27551985, 0.2583293 ,
       0.24426825, 0.2732193 , 0.23977351, 0.23207869, 0.2239526 ,
       0.21523343, 0.25460303, 0.27394077, 0.23523484, 0.23723707,
       0.222791  , 0.18889476, 0.21227987, 0.2113625 , 0.2202905 ,
       0.2670882 , 0.26453573, 0.26543015, 0.28290066, 0.27036318,
       0.28161994, 0.23983459, 0.2399843 , 0.2538837 , 0.28617197],
      dtype=float32)
                },
                'post': {
                    'b': array([ 0.00227712,  0.00410922,  0.0058516 , ...,  0.00019771,
       -0.00454297,  0.00112776], dtype=float32),
                    'w': array([[[ 1.60219464e-02, -6.32830570e-03, -4.34190384e-04, ...,
         -2.05251835e-02,  5.95892640e-03, -4.76127584e-03],
        [-9.98676405e-05, -7.33074406e-03, -1.26863532e-02, ...,
          7.18150195e-03, -1.75036013e-03,  1.55169265e-02],
        [-5.81251038e-03, -5.87717863e-03,  2.19022054e-02, ...,
         -5.58457756e-03,  1.16657428e-02, -1.81494025e-03],
        ...,
        [-7.13085895e-03,  2.03952119e-02, -9.58707742e-03, ...,
         -1.69843007e-02,  9.77954641e-03, -2.47077830e-03],
        [-1.87334344e-02, -2.76322174e-03,  2.28968691e-02, ...,
         -3.18435661e-04,  1.49174239e-02, -2.33985633e-02],
        [-8.63831770e-03, -4.08300431e-03,  2.85220704e-05, ...,
          2.67661619e-03, -9.26318858e-03, -3.84686440e-02]],

[[-4.41310491e-04, -1.40345162e-02,  1.91439223e-02, ...,
         -2.57955100e-02, -3.95066477e-03, -5.66026196e-03],
        [ 2.54288558e-02, -9.75304935e-03, -1.79422498e-02, ...,
          1.74219143e-02, -1.06191719e-02, -6.14397135e-03],
        [ 6.09810324e-03,  1.33533711e-02, -1.90592080e-03, ...,
          3.46733183e-02, -6.68695569e-03,  1.98827628e-02],
        ...,
        [ 6.94470434e-03,  3.06668086e-03,  2.26073414e-02, ...,
          9.81861353e-03,  1.64711662e-02,  1.25699742e-02],
        [ 4.30469355e-03,  5.68447029e-03,  6.36044890e-03, ...,
         -2.36831773e-02,  1.19379545e-02, -5.29726502e-03],
        [ 2.51850276e-03,  9.26471315e-03,  9.80320293e-03, ...,
         -1.59647278e-02, -1.53232217e-02,  1.45909069e-02]],

[[-3.06620151e-02, -2.45522999e-04,  6.68670982e-03, ...,
          2.02747677e-02,  5.18184528e-03, -2.45395079e-02],
        [ 7.14192027e-03, -3.25411628e-03,  1.59157347e-03, ...,
          1.82876887e-03,  8.23392812e-03,  1.77870430e-02],
        [-1.15546295e-02,  6.07956946e-03, -1.10001322e-02, ...,
         -1.87739674e-02, -1.17661315e-03, -1.34597709e-02],
        ...,
        [ 9.34998877e-03, -5.40582417e-03, -1.88018735e-02, ...,
          4.86738893e-04,  1.34934839e-02, -2.02603196e-03],
        [ 1.87009713e-03,  2.01993808e-02,  3.71760130e-03, ...,
         -1.92958973e-02, -2.36028638e-02,  2.66806595e-03],
        [-1.90301463e-02, -2.56970990e-03,  1.77047886e-02, ...,
          4.61818185e-03,  6.54727104e-04, -2.83123256e-04]],

...,

[[-8.25279579e-03, -2.55150124e-02,  1.30663087e-04, ...,
         -6.57988945e-03,  1.55982636e-02, -2.05582269e-02],
        [-1.54422820e-02,  1.02187982e-02, -5.20728237e-04, ...,
          5.83955599e-03, -9.00995336e-04,  2.20121536e-02],
        [-3.67736374e-03,  4.01381217e-03, -1.65678114e-02, ...,
         -9.28543974e-03, -2.38824124e-03, -2.97486270e-03],
        ...,
        [ 1.53934481e-02,  5.41916164e-03,  6.68860646e-03, ...,
          1.36334402e-02,  4.15810803e-03,  8.54777079e-03],
        [-1.05572389e-02,  1.10236940e-03, -1.56435389e-02, ...,
         -1.09226611e-02, -2.01439951e-02,  5.28248446e-03],
        [ 1.40633499e-02,  5.62130706e-04,  1.23525541e-02, ...,
         -2.03714464e-02,  2.83894949e-02,  1.10108349e-02]],

[[-5.71528636e-03, -1.33396322e-02,  4.63643577e-03, ...,
         -1.61747616e-02,  6.38877798e-04, -3.66402161e-03],
        [ 1.30788377e-02, -3.41259711e-03, -1.25344051e-02, ...,
          1.28065860e-02, -7.42243545e-04,  4.35881084e-03],
        [-6.52339077e-03, -1.38171408e-02,  1.74011346e-02, ...,
          7.57731358e-03, -1.02070905e-02, -6.46297028e-03],
        ...,
        [ 2.43425788e-03,  1.11944843e-02,  6.44581951e-03, ...,
          1.18476069e-02, -4.20171767e-03, -1.02401907e-02],
        [ 1.09890383e-02, -1.76882893e-02,  9.50387772e-03, ...,
          3.37445876e-03,  7.73787207e-04, -5.39443130e-03],
        [-4.42360324e-04,  2.67955121e-02, -4.17449512e-03, ...,
          1.33155966e-02, -1.50912609e-02, -1.35448817e-02]],

[[-5.88139240e-03, -2.03647055e-02, -1.63833331e-02, ...,
         -1.43157914e-02, -2.27124058e-02, -1.14719691e-02],
        [ 1.83314341e-03, -2.54484490e-02, -8.61003343e-03, ...,
          4.26080049e-04, -1.36934174e-02,  1.51861394e-02],
        [-1.15379028e-03, -3.19069833e-03, -2.13307329e-03, ...,
         -2.33514644e-02,  2.00367980e-02, -6.51050312e-03],
        ...,
        [ 7.55283562e-03, -9.96225234e-03,  1.34144872e-02, ...,
          6.54450199e-03, -3.36529757e-03,  5.97001240e-03],
        [ 3.44229583e-03,  9.81338508e-03, -1.04225297e-02, ...,
         -9.26869828e-03, -3.25994082e-02, -1.25933774e-02],
        [ 1.09423827e-02, -1.90035310e-02,  6.49145385e-03, ...,
          3.46495374e-03,  9.23398882e-03,  1.87372081e-02]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.44482347, 0.42007974, 0.42201445, ..., 0.41606164,
         0.42287317, 0.4234973 ],
        [0.39234987, 0.42474   , 0.40693808, ..., 0.4075191 ,
         0.42756715, 0.4116632 ],
        [0.43525642, 0.41221023, 0.41619653, ..., 0.3993511 ,
         0.4090455 , 0.38683847],
        ...,
        [0.37275848, 0.36328268, 0.3754888 , ..., 0.333638  ,
         0.3948719 , 0.38432208],
        [0.43789008, 0.433027  , 0.41810986, ..., 0.41066962,
         0.40822017, 0.44112036],
        [0.4075499 , 0.42448154, 0.41999772, ..., 0.4200564 ,
         0.40848023, 0.43024692]]], dtype=float32)
                },
                'query': {
                    'b': array([[ 0.00630543, -0.01040035, -0.00061541, ...,  0.00912199,
        -0.00037946, -0.00610306],
       [ 0.01307749, -0.0054321 ,  0.00767672, ...,  0.0043904 ,
         0.00037232,  0.01104123],
       [ 0.01931557,  0.00255344, -0.01728779, ..., -0.00125953,
         0.00231459,  0.02259091],
       ...,
       [ 0.00335209,  0.01204891, -0.02524816, ..., -0.00921736,
        -0.00198141,  0.01285304],
       [ 0.03694161, -0.01562951, -0.00046486, ...,  0.01202064,
         0.02256216,  0.0066755 ],
       [-0.01016504,  0.01377161,  0.01127281, ..., -0.01075078,
         0.01636219, -0.00281313]], dtype=float32),
                    'w': array([[[ 0.00683608,  0.01803902, -0.0127693 , ..., -0.00610208,
          0.05388   ,  0.00311598],
        [-0.01006073, -0.00214602,  0.01311889, ..., -0.02653091,
          0.00350361,  0.00938338],
        [-0.02756437, -0.03875   , -0.01696985, ..., -0.00325368,
         -0.04992706,  0.03636185],
        ...,
        [ 0.04123638,  0.03804113, -0.03609247, ..., -0.0377653 ,
         -0.01802078, -0.00413028],
        [ 0.01214722,  0.01488034,  0.02144075, ...,  0.01663101,
         -0.00706379, -0.00448915],
        [-0.01851939, -0.00234487, -0.02068215, ..., -0.00288961,
          0.04119675,  0.02511111]],

[[ 0.00617579, -0.0171384 ,  0.00686683, ..., -0.04633516,
          0.03434372, -0.04915209],
        [ 0.00292073, -0.0134772 , -0.0130819 , ...,  0.02139365,
         -0.01328588, -0.02387512],
        [ 0.01560103, -0.01468205, -0.00886597, ...,  0.00768105,
          0.00749326, -0.00756207],
        ...,
        [ 0.00292776,  0.01112255, -0.00232512, ..., -0.02031175,
         -0.01799656,  0.0309556 ],
        [ 0.01065346, -0.04277091,  0.0333715 , ...,  0.02087235,
         -0.00745687, -0.03341635],
        [-0.01501742,  0.02088866, -0.01778351, ..., -0.03369918,
         -0.00626652,  0.01321735]],

[[-0.0055393 , -0.00236754, -0.00410342, ...,  0.0211159 ,
          0.0147532 ,  0.0572011 ],
        [-0.00275519, -0.03825096,  0.02899468, ..., -0.01704872,
         -0.0140565 , -0.02606585],
        [ 0.00060486,  0.00104801, -0.00801584, ..., -0.05260897,
          0.00465189,  0.00236722],
        ...,
        [ 0.01045739,  0.0190653 , -0.00023837, ...,  0.00201208,
         -0.01351035,  0.0236369 ],
        [ 0.00746405, -0.00309808, -0.01823745, ...,  0.0304853 ,
          0.00970365, -0.00635394],
        [ 0.00386312, -0.00292712,  0.00801195, ...,  0.00039294,
         -0.0146638 , -0.01207684]],

...,

[[-0.0031831 ,  0.00060943, -0.00174922, ...,  0.02849803,
         -0.00629389,  0.01923761],
        [ 0.00018168, -0.04518702,  0.01124255, ..., -0.01228412,
         -0.01699694, -0.01914257],
        [ 0.01863275, -0.01569023, -0.0041512 , ...,  0.02775211,
          0.01504739, -0.01101425],
        ...,
        [-0.00861158,  0.00541674,  0.02368934, ...,  0.01441211,
          0.0306981 , -0.01107702],
        [ 0.00709426, -0.02020066,  0.01229627, ...,  0.02485121,
         -0.02080423, -0.00478564],
        [-0.01539185, -0.02087969,  0.0006622 , ...,  0.00759443,
         -0.0037208 , -0.04737531]],

[[-0.00341126, -0.0439684 ,  0.006416  , ...,  0.02101934,
         -0.05221625,  0.01797302],
        [ 0.00762315,  0.01026404,  0.00066357, ..., -0.05038482,
         -0.03188071,  0.00532991],
        [-0.01031536,  0.03580642, -0.02827327, ...,  0.02861057,
         -0.00060704,  0.00086264],
        ...,
        [-0.00888584, -0.01304739,  0.00629479, ..., -0.01062987,
         -0.00725211,  0.00968421],
        [ 0.02497604, -0.02214685,  0.02492903, ..., -0.00038655,
         -0.02753083, -0.02061361],
        [ 0.0233178 , -0.01245518, -0.00924736, ..., -0.0195448 ,
          0.00933027,  0.01638415]],

[[-0.02320945,  0.03080179, -0.01184121, ..., -0.02594195,
         -0.0131538 , -0.01832458],
        [-0.00467245,  0.03966678, -0.00394025, ...,  0.02598585,
          0.01235735,  0.02955044],
        [-0.03318202, -0.00447797,  0.00395965, ...,  0.02871543,
          0.0354715 , -0.03632242],
        ...,
        [ 0.02489721,  0.01103833, -0.00312272, ..., -0.00588333,
          0.02076965,  0.01153785],
        [ 0.01564988, -0.01428127, -0.01250215, ..., -0.01380367,
         -0.01107514, -0.04016251],
        [ 0.03295957, -0.00897606, -0.01479999, ...,  0.01106019,
         -0.01252355,  0.01575936]]], dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.698365  , 0.71223074, 0.86259055, ..., 0.7946437 ,
         0.76687956, 0.8516118 ],
        [0.68646514, 0.7379805 , 0.7123095 , ..., 0.7697349 ,
         0.71448344, 0.80369186],
        [0.88516766, 0.719584  , 0.84331304, ..., 0.8656582 ,
         0.78091383, 0.90902406],
        ...,
        [0.54961836, 0.49563563, 0.5916302 , ..., 0.5680633 ,
         0.57030255, 0.66667366],
        [0.76514006, 0.7269518 , 0.804479  , ..., 0.78656787,
         0.6813261 , 0.6139957 ],
        [0.6441092 , 0.68763   , 0.7174395 , ..., 0.6869405 ,
         0.60524416, 0.7539055 ]]], dtype=float32)
                },
                'value': {
                    'b': array([[-1.6719506e-03,  3.1838890e-03, -3.4906643e-03, ...,
         2.1193527e-04, -1.3315614e-03, -2.9562660e-03],
       [-3.5157211e-03,  5.4036635e-03, -5.3099038e-06, ...,
        -1.5847052e-03,  1.8464231e-04,  7.0413254e-04],
       [-2.6758336e-03, -2.0398316e-03, -1.1476204e-03, ...,
        -5.6534848e-04, -1.3752164e-04, -2.1303194e-03],
       ...,
       [ 5.4894928e-03, -6.2388863e-04, -1.8957579e-03, ...,
         5.6731803e-03,  3.9430051e-03, -4.2474028e-03],
       [-2.0500964e-03,  1.6432669e-03, -4.0994561e-03, ...,
        -1.0819612e-03,  7.0421447e-05,  2.1111206e-03],
       [-5.5974408e-04,  3.3670918e-03,  1.4440225e-03, ...,
        -2.9739656e-04,  2.3998234e-03,  3.0667472e-03]], dtype=float32),
                    'w': array([[[-1.78365000e-02,  2.84314482e-03, -1.39748259e-02, ...,
          1.78501215e-02,  9.85812745e-04, -9.48785862e-04],
        [ 7.44687626e-03,  4.15558752e-04, -4.96728625e-03, ...,
          4.96215979e-03,  1.66751747e-03, -1.90461092e-02],
        [ 1.76583678e-02,  4.45447396e-03, -2.55572014e-02, ...,
          1.62308514e-02,  3.32947746e-02,  2.39235219e-02],
        ...,
        [ 2.78520212e-03,  1.96315497e-02,  6.07665209e-03, ...,
          1.57222468e-02,  3.04588443e-03, -4.29826090e-04],
        [-7.40560528e-04,  7.82942306e-03, -5.32508362e-03, ...,
         -2.35469788e-02, -3.91680887e-03,  1.02615301e-02],
        [-2.59020971e-03,  1.15150996e-02,  1.41894817e-03, ...,
          2.16003098e-02, -1.32964994e-03,  2.13793032e-02]],

[[ 1.95347611e-02,  4.09426121e-03, -7.18522677e-03, ...,
         -7.35512329e-03, -2.51324032e-03,  2.10163761e-02],
        [-1.22168576e-02,  1.16148479e-02, -1.25418408e-02, ...,
         -1.39319524e-02, -3.29393358e-03,  9.16428398e-03],
        [-6.41297520e-05, -1.89825390e-02,  7.18786381e-03, ...,
         -1.52531229e-02,  8.93052388e-03,  8.45586695e-03],
        ...,
        [-1.71794575e-02, -7.80098664e-04, -2.68197656e-02, ...,
         -2.68879486e-03, -1.19984448e-02, -1.29080508e-02],
        [ 1.89863853e-02,  5.97108714e-03,  1.72898476e-03, ...,
          1.93750951e-03, -1.89724786e-03,  2.40854193e-02],
        [ 3.44004296e-03,  2.41522342e-02, -6.75652968e-03, ...,
          3.05757988e-02, -1.76970137e-03,  2.39947755e-02]],

[[-2.36936510e-02, -9.54832323e-03, -1.24944001e-02, ...,
         -1.15259392e-02,  6.53631194e-03,  2.14831289e-02],
        [-6.42117858e-03,  1.27027715e-02,  7.31832301e-03, ...,
          1.37724392e-02,  6.83144259e-04, -6.80594007e-03],
        [-3.28513957e-03,  2.78529804e-03,  3.04201376e-02, ...,
          9.64713562e-03,  4.57322516e-04,  2.34081876e-02],
        ...,
        [ 9.35432594e-03, -4.19412740e-04, -7.96502829e-03, ...,
         -2.37759529e-03,  1.59879017e-03,  2.62604700e-03],
        [-1.22024957e-03, -6.62017195e-03,  1.33248037e-02, ...,
          1.86617821e-02,  2.16839593e-02,  5.24432166e-03],
        [ 6.06797030e-03,  3.43944021e-02,  2.02117581e-03, ...,
          1.13977827e-02, -6.66222861e-03,  3.11012752e-03]],

...,

[[ 5.20108826e-03, -3.11744818e-03, -1.66065153e-02, ...,
         -7.78459362e-04, -9.26640909e-03,  2.23218813e-03],
        [-8.02782085e-03,  3.90778005e-04,  1.40807265e-02, ...,
          1.04253488e-02,  1.02921100e-02, -1.39354472e-03],
        [-2.30831001e-02, -1.70340925e-03,  1.70863736e-02, ...,
         -1.05764112e-03, -1.25060035e-02, -1.86898606e-03],
        ...,
        [ 3.22063453e-03, -8.41359980e-03, -8.37549847e-03, ...,
         -5.70000801e-03, -6.99548473e-05,  1.13312490e-02],
        [ 6.91914652e-03, -1.15573416e-02,  8.06923956e-03, ...,
          5.88306691e-03,  1.21457176e-02,  4.20463085e-03],
        [-2.13251878e-02,  6.74338918e-03, -2.55054212e-03, ...,
          6.26634760e-03,  1.27693638e-02, -9.49179009e-03]],

[[-2.28971103e-03, -1.59675181e-02, -1.96324103e-02, ...,
         -1.09331785e-02, -1.80871077e-02, -9.93164955e-04],
        [-3.15288501e-03, -2.21888702e-02,  1.62585720e-03, ...,
         -6.58442220e-03,  1.75409596e-02, -2.85189990e-02],
        [-5.46636432e-03,  1.43901836e-02, -4.32401896e-02, ...,
          8.08360800e-03,  2.42605549e-03,  9.65884980e-03],
        ...,
        [ 1.23602003e-02, -8.04022979e-03,  2.26687081e-03, ...,
          2.02171206e-02, -3.39716999e-03,  8.49399774e-04],
        [-1.99223286e-03, -2.96767824e-03,  5.06375625e-04, ...,
          2.97836587e-02, -1.72054756e-03, -1.06025031e-02],
        [-1.21570658e-02, -9.32336412e-03, -6.03768835e-03, ...,
         -4.79857996e-03,  2.16034781e-02,  1.69795938e-04]],

[[-1.36184208e-02,  1.36335846e-02,  7.44790910e-03, ...,
          2.19548345e-02,  6.67087268e-03,  1.30617796e-02],
        [-8.48605391e-03,  1.20121036e-02,  7.09828734e-03, ...,
          2.65877750e-02, -6.88361761e-04, -8.83432850e-03],
        [ 2.57309317e-03,  1.95993576e-03,  1.10754780e-02, ...,
          3.56037379e-03,  3.57526354e-03, -2.06974614e-02],
        ...,
        [-4.41282243e-03,  3.95893073e-03, -1.09358151e-02, ...,
          7.15045398e-03,  6.64680032e-03, -1.82183227e-03],
        [ 1.63184218e-02, -9.19388887e-03,  3.04607246e-02, ...,
          1.72292423e-02, -2.24216143e-03,  1.58525407e-02],
        [ 1.88137626e-03, -1.31328674e-02,  2.44586747e-02, ...,
         -1.11989332e-02, -4.85838484e-03,  1.58309378e-02]]],
      dtype=float32),
                    'lora_a': Array([[-0.10846964],
       [ 0.29453316],
       [-0.2603706 ],
       ...,
       [-0.7982575 ],
       [ 0.7376351 ],
       [-0.08402041]], dtype=float32),
                    'lora_b': Array([[0.],
       [0.],
       [0.],
       ...,
       [0.],
       [0.],
       [0.]], dtype=float32),
                    'dora_m': Array([[[0.48664016, 0.48282182, 0.47897825, ..., 0.4621614 ,
         0.46521682, 0.4696118 ],
        [0.43445462, 0.4764754 , 0.45074508, ..., 0.45639613,
         0.46762112, 0.4579986 ],
        [0.4773432 , 0.44959944, 0.46149546, ..., 0.46104416,
         0.4564243 , 0.45269182],
        ...,
        [0.40127963, 0.44633105, 0.44957897, ..., 0.3298088 ,
         0.46749434, 0.42867705],
        [0.4912154 , 0.4943047 , 0.4536157 , ..., 0.46936134,
         0.4902126 , 0.47923133],
        [0.4688782 , 0.4799055 , 0.4766657 , ..., 0.5002816 ,
         0.49480858, 0.48734224]]], dtype=float32)
                }
            }
        }
    }
}

Changed text

Open file

{
    'freq_emb': {
        'emb_var': Array([[ 0.05383475, -0.02290652, -0.02175547, ..., -0.06938078,
         0.0495423 ,  0.00398525],
       [ 0.04227516, -0.02249096,  0.05041242, ...,  0.01230512,
        -0.03456249, -0.10940999],
       [-0.06831896,  0.14380653, -0.04166476, ...,  0.11798981,
         0.11934441,  0.00524181]], dtype=float32)
    },
    'horizon_ff_layer': {
        'hidden_layer': {
            'bias': {
                'b': Array([-0.01867074, -0.00871083, -0.02053265, ..., -0.00441845,
       -0.01340919, -0.00125625], dtype=float32)
            },
            'linear': {
                'w': Array([[-0.00122285,  0.04531502, -0.01212596, ..., -0.00281182,
         0.02374083,  0.02576321],
       [-0.0382062 ,  0.0059988 , -0.02117914, ...,  0.06201293,
        -0.00809431,  0.00410271],
       [-0.00531332,  0.00811246,  0.00494867, ..., -0.04757487,
        -0.00602691, -0.00266292],
       ...,
       [ 0.03347114,  0.03805783, -0.013433  , ...,  0.01496901,
         0.00796273,  0.06580582],
       [-0.04087714, -0.00858165, -0.06531095, ..., -0.03156528,
         0.0158751 , -0.01732916],
       [ 0.05645538, -0.01491658, -0.03915612, ...,  0.01406249,
        -0.02100163,  0.01975475]], dtype=float32)
            }
        },
        'output_layer': {
            'bias': {
                'b': Array([-6.3380185e-03, -9.0496093e-03, -4.2588129e-03, ...,
       -4.3446473e-03, -6.0337975e-06, -3.0872226e-03], dtype=float32)
            },
            'linear': {
                'w': Array([[-0.05310262, -0.02206682,  0.0271988 , ..., -0.01757539,
        -0.02656861,  0.00936702],
       [ 0.00877119, -0.0578129 , -0.03252782, ..., -0.0356226 ,
        -0.02773722, -0.02957312],
       [-0.0370421 ,  0.00511809, -0.05719691, ...,  0.00449829,
        -0.02505858, -0.02064183],
       ...,
       [ 0.05921374,  0.08082739,  0.08133432, ..., -0.03276784,
        -0.05346885, -0.00108078],
       [ 0.02845641,  0.03454759,  0.02899572, ..., -0.03797804,
        -0.00647498, -0.0080282 ],
       [-0.06980696, -0.065199  , -0.09629356, ..., -0.01494522,
        -0.00420657,  0.03861954]], dtype=float32)
            }
        },
        'residual_layer': {
            'bias': {
                'b': Array([-0.00071009, -0.00952863, -0.01032941, ...,  0.00253608,
        0.00484881, -0.00057579], dtype=float32)
            },
            'linear': {
                'w': Array([[-8.0529033e-05, -8.1122480e-03, -1.1188935e-02, ...,
         2.7386495e-03,  4.1057365e-03,  3.1871016e-03],
       [-2.5267012e-02, -2.9770406e-02, -2.1402119e-02, ...,
         9.0102842e-03,  1.3215628e-02,  8.5849436e-03],
       [-3.2064769e-02, -9.2049723e-04, -8.8477237e-03, ...,
        -2.2026885e-02, -2.9883629e-02, -4.2654805e-02],
       ...,
       [-4.1601930e-02, -1.1100663e-02, -2.1980958e-02, ...,
         3.9413955e-02,  4.5028899e-02,  4.5041710e-02],
       [ 8.9835078e-03,  1.0722069e-03, -1.1620998e-02, ...,
        -3.4306385e-02, -2.8038949e-02, -2.8952893e-02],
       [-1.2415100e-02, -2.3502124e-02, -1.4703525e-02, ...,
        -2.9797151e-03,  1.5994614e-03,  8.6680241e-04]], dtype=float32)
            }
        }
    },
    'input_ff_layer': {
        'hidden_layer': {
            'bias': {
                'b': Array([-0.01302247, -0.0093135 , -0.00869382, ..., -0.02097026,
       -0.00173573, -0.00588994], dtype=float32)
            },
            'linear': {
                'w': Array([[-0.02687993, -0.03276388,  0.0118101 , ...,  0.01630807,
         0.02635014,  0.01112787],
       [ 0.03501734,  0.03296082, -0.00576759, ...,  0.01727151,
        -0.04001599, -0.00386006],
       [ 0.04767988,  0.01772386,  0.04430255, ..., -0.00390798,
        -0.02813526,  0.02526983],
       ...,
       [ 0.00559828, -0.02095459,  0.01350581, ...,  0.01727615,
         0.03876091,  0.01288361],
       [-0.04138084,  0.03345452, -0.00055608, ...,  0.06449523,
         0.02750022, -0.01134627],
       [-0.04997209,  0.05873165,  0.05058305, ..., -0.01771185,
        -0.04247753, -0.01341925]], dtype=float32)
            }
        },
        'output_layer': {
            'bias': {
                'b': Array([-0.0011562 ,  0.00012631,  0.00202118, ...,  0.00015547,
        0.00136226, -0.00038068], dtype=float32)
            },
            'linear': {
                'w': Array([[-0.01875325,  0.01847108, -0.03379152, ...,  0.01269644,
         0.00307103, -0.00174853],
       [ 0.06125693, -0.00906967, -0.03221472, ...,  0.03700627,
        -0.03955566,  0.0085879 ],
       [-0.01743414,  0.04761524, -0.01841931, ..., -0.03325716,
         0.01393927,  0.0339308 ],
       ...,
       [-0.00523872, -0.03900931,  0.01258594, ...,  0.05073977,
         0.0148632 , -0.00127181],
       [ 0.00715443,  0.01426611, -0.02727203, ...,  0.01418048,
        -0.02707596, -0.01978715],
       [-0.0379927 ,  0.04774408,  0.01647678, ..., -0.0322981 ,
        -0.02338383, -0.00932761]], dtype=float32)
            }
        },
        'residual_layer': {
            'bias': {
                'b': Array([-0.00342117,  0.0011725 , -0.00115519, ..., -0.01340925,
       -0.01195502, -0.0078334 ], dtype=float32)
            },
            'linear': {
                'w': Array([[ 0.060296  ,  0.05077375,  0.06998359, ..., -0.03257499,
         0.05363302, -0.049019  ],
       [ 0.03054528, -0.03217815, -0.0089082 , ..., -0.04977859,
        -0.02510578,  0.01701915],
       [-0.04686454,  0.03704751, -0.02275803, ...,  0.0496948 ,
         0.03227101,  0.0320745 ],
       ...,
       [ 0.07358721,  0.05074072,  0.06101955, ...,  0.0217406 ,
        -0.00013694,  0.01635611],
       [ 0.04090381,  0.07156919,  0.00553412, ..., -0.04919641,
         0.05928294, -0.05756657],
       [-0.03001764, -0.01901046,  0.0523936 , ..., -0.05530353,
         0.04039487, -0.01157674]], dtype=float32)
            }
        }
    },
    'stacked_transformer_layer': {
        'x_layers_0': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': Array([ 0.00422558,  0.00153717, -0.00328633, ...,  0.01313652,
        0.01618191,  0.0046046 ], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.0408473 , 0.99524635, 0.98682207, ..., 1.1109077 , 1.0122046 ,
        1.0305351 ]], dtype=float32),
                        'lora_a': Array([[-0.10847064],
       [ 0.294551  ],
       [-0.26035047],
       ...,
       [-0.79824907],
       [ 0.73761564],
       [-0.08403018]], dtype=float32),
                        'lora_b': Array([[ 5.1776869e-06],
       [ 1.5813343e-05],
       [-1.4205739e-05],
       ...,
       [-9.6916729e-06],
       [-1.8683011e-06],
       [-5.3805843e-06]], dtype=float32),
                        'w': Array([[-0.04934547,  0.03615815,  0.02133694, ...,  0.00294278,
         0.03395786, -0.02778906],
       [-0.00490594,  0.03894801, -0.02373052, ..., -0.03631651,
         0.0288607 ,  0.05464533],
       [-0.00822064, -0.0459388 ,  0.02931811, ...,  0.0401447 ,
        -0.0274516 , -0.0161763 ],
       ...,
       [-0.01827895,  0.05226748,  0.01714141, ..., -0.005661  ,
         0.02436945,  0.01169135],
       [-0.03512175,  0.0176127 ,  0.02929639, ...,  0.0022374 ,
         0.01826224,  0.03047828],
       [-0.00179529, -0.01935859,  0.02216526, ..., -0.01494863,
         0.04285045, -0.02584894]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': Array([ 8.5014108e-05, -3.6904723e-03, -2.2362696e-03, ...,
       -2.6434378e-03, -4.6255854e-03, -2.7873858e-03], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.0201988, 1.0224365, 1.0201229, ..., 1.0271175, 1.0138276,
        1.0606217]], dtype=float32),
                        'lora_a': Array([[-0.10848837],
       [ 0.29451126],
       [-0.26038843],
       ...,
       [-0.7982782 ],
       [ 0.737633  ],
       [-0.08402973]], dtype=float32),
                        'lora_b': Array([[ 9.4581455e-06],
       [-5.8203859e-06],
       [-4.4713113e-07],
       ...,
       [ 1.9027753e-05],
       [-4.2390179e-06],
       [-1.8873763e-05]], dtype=float32),
                        'w': Array([[-0.00317632,  0.01268681, -0.02180147, ..., -0.03509863,
         0.02900657,  0.05286129],
       [ 0.03748895, -0.01666473,  0.04159482, ...,  0.01672644,
        -0.03867533,  0.03111953],
       [-0.04813586, -0.03120456,  0.02479169, ...,  0.01334335,
        -0.04884216,  0.00229996],
       ...,
       [-0.03854724,  0.0137933 , -0.01018869, ..., -0.0093796 ,
         0.03359092,  0.00674478],
       [ 0.0059021 , -0.00476607, -0.01634962, ...,  0.02304816,
        -0.01485625,  0.01752446],
       [-0.02087662,  0.00162849,  0.0099754 , ..., -0.06226157,
         0.02198281, -0.00740848]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': Array([ 0.00631881,  0.01150341,  0.00914302, ..., -0.0200112 ,
       -0.00163537, -0.000495  ], dtype=float32),
                    'scale': Array([-0.03739248, -0.0243671 , -0.02632998, ..., -0.01486455,
        0.01445367,  0.04890316], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': Array([1.2228152 , 1.1309394 , 1.2290405 , ..., 0.96604544, 0.9487428 ,
       0.9532705 ], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': Array([[-6.4075930e-06,  4.2504727e-04, -1.1618993e-03, ...,
        -1.7168164e-03,  2.2836959e-04, -8.8550907e-05],
       [ 3.1805394e-04,  3.4957723e-04,  5.1292509e-04, ...,
         2.6313388e-03,  1.0807015e-03, -2.1224995e-03],
       [ 9.5978525e-04, -3.6766953e-03, -1.1368678e-03, ...,
         1.4873255e-03, -2.3283053e-03, -4.0190420e-03],
       ...,
       [-1.6945685e-03, -1.5151855e-03, -1.4960926e-03, ...,
         1.8141167e-03, -2.3590028e-03, -2.6155929e-03],
       [-5.4188799e-03,  4.6130762e-04, -6.1691484e-05, ...,
         3.2484343e-03, -2.7115741e-03, -3.6162001e-04],
       [-2.8124801e-03,  1.5307369e-04, -2.2465425e-05, ...,
         9.6123409e-04, -2.0123613e-03, -4.0718642e-05]], dtype=float32),
                    'dora_m': Array([[[0.29071984, 0.29683936, 0.30372328, ..., 0.34602427,
         0.29602143, 0.305746  ],
        [0.32390478, 0.33643064, 0.34294572, ..., 0.43285245,
         0.3494392 , 0.33882546],
        [0.37621891, 0.43592635, 0.43361625, ..., 0.48496065,
         0.41900933, 0.39550942],
        ...,
        [0.6301863 , 0.8522955 , 0.5222311 , ..., 0.4287274 ,
         0.5746888 , 0.44996235],
        [0.53644997, 0.38934904, 0.48470625, ..., 0.5750841 ,
         0.32962137, 0.76050806],
        [0.3797709 , 0.32443294, 0.3033168 , ..., 0.2547274 ,
         0.25814548, 0.25158834]]], dtype=float32),
                    'lora_a': Array([[-0.10848147],
       [ 0.2945259 ],
       [-0.26038498],
       ...,
       [-0.79826516],
       [ 0.73763   ],
       [-0.08401564]], dtype=float32),
                    'lora_b': Array([[ 5.0566196e-06],
       [-1.0148730e-05],
       [ 1.1051195e-05],
       ...,
       [ 6.9015719e-06],
       [ 7.3781512e-06],
       [-1.4809251e-05]], dtype=float32),
                    'w': Array([[[-2.00603511e-02,  5.75137278e-03, -8.72080214e-03, ...,
         -2.70311963e-02,  1.24247791e-03, -2.36301706e-03],
        [ 3.99277313e-03,  2.94813123e-02, -1.99544337e-02, ...,
          1.95915089e-03,  1.23697640e-02,  5.81120653e-03],
        [-1.01987063e-03,  3.53949121e-03, -6.74259616e-03, ...,
          3.45695938e-04, -3.17200739e-03,  1.25762047e-02],
        ...,
        [ 1.12090185e-02,  5.59417754e-02,  2.80567389e-02, ...,
          5.72427809e-02, -2.45896727e-02, -4.41132709e-02],
        [-2.39311568e-02,  1.08251497e-02,  1.98488291e-02, ...,
          2.62170825e-02, -1.07316952e-02,  7.13963956e-02],
        [-5.38694439e-03,  2.10208986e-02,  4.16014344e-03, ...,
          1.06410738e-02,  1.50726573e-03, -1.23633593e-02]],

...,

[[-1.08604589e-02,  1.28442254e-02,  5.53134922e-03, ...,
         -5.09914523e-03, -1.28980596e-02,  1.25545133e-02],
        [-1.18277026e-02,  8.53359420e-03,  3.05717532e-03, ...,
          1.41720111e-02, -1.60324126e-02,  1.07069116e-03],
        [-1.69687124e-03,  8.45217053e-03,  6.06904272e-03, ...,
         -1.82976536e-02,  7.91715831e-03,  2.23788200e-03],
        ...,
        [-6.64934842e-03,  6.04555802e-03, -1.44436248e-02, ...,
         -7.54461996e-03,  1.34342900e-02,  1.26932003e-03],
        [ 8.17108434e-03,  2.05422216e-03,  1.13008972e-02, ...,
         -1.15924431e-02,  7.23493379e-03, -1.01354849e-02],
        [-3.08707799e-03, -2.84547848e-03,  5.24008553e-03, ...,
         -9.74707212e-03, -9.47374292e-03,  8.77883460e-04]]],      dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': Array([0.11543996, 0.19667041, 0.08449101, 0.15537748, 0.05297864,
       0.08272862, 0.08896066, 0.09299482, 0.14027727, 0.12275486,
       0.13754928, 0.11227061, 0.07794799, 0.11385844, 0.21155985,
       0.07013194, 0.11293491, 0.11814845, 0.14049305, 0.1071273 ,
       0.07294399, 0.11549649, 0.10932729, 0.03889112, 0.07322115,
       0.09397113, 0.08638889, 0.086013  , 0.09734396, 0.04915351,
       0.07681435, 0.0727135 , 0.06417551, 0.1102055 , 0.05094118,
       0.06759506, 0.10526094, 0.06561205, 0.08017935, 0.08598939,
       0.08295949, 0.10986121, 0.06075855, 0.06381149, 0.09573651,
       0.15352455, 0.10654029, 0.05720843, 0.1080655 , 0.08904675,
       0.11533722, 0.10753154, 0.14174831, 0.05750328, 0.10738507,
       0.13763388, 0.11135354, 0.07917753, 0.05568942, 0.18191479,
       0.19961604, 0.08848751, 0.1105781 , 0.11773358, 0.13192818,
       0.05453964, 0.07397411, 0.04283143, 0.09388805, 0.12555331,
       0.15708339, 0.14256129, 0.08874376, 0.08300523, 0.08548183,
       0.07495448, 0.01806459, 0.10216796, 0.08380181, 0.06248857],      dtype=float32)
                },
                'post': {
                    'b': Array([ 0.0038859 ,  0.00227933,  0.00126942, ..., -0.00807348,
       -0.0033168 , -0.00190115], dtype=float32),
                    'dora_m': Array([[[0.19123024, 0.14076011, 0.14161648, ..., 0.15473266,
         0.1719282 , 0.14289714],
        [0.22512862, 0.25811967, 0.23838049, ..., 0.25743642,
         0.24412851, 0.26848155],
        [0.15662938, 0.15710527, 0.16476132, ..., 0.13925603,
         0.13652036, 0.15013184],
        ...,
        [0.31857708, 0.30544275, 0.28747907, ..., 0.2713666 ,
         0.29232302, 0.2772732 ],
        [0.29738736, 0.26313943, 0.25538403, ..., 0.27492785,
         0.28833735, 0.2847642 ],
        [0.2633174 , 0.26364738, 0.2317865 , ..., 0.2222713 ,
         0.26653096, 0.24285616]]], dtype=float32),
                    'lora_a': Array([[-0.10847466],
       [ 0.29454225],
       [-0.2603705 ],
       ...,
       [-0.7982707 ],
       [ 0.7376422 ],
       [-0.08400187]], dtype=float32),
                    'lora_b': Array([[ 1.2972054e-05],
       [-3.7403761e-06],
       [ 6.7211336e-06],
       ...,
       [ 1.2964535e-05],
       [ 1.6402071e-05],
       [ 1.2927146e-05]], dtype=float32),
                    'w': Array([[[ 0.01001332,  0.00383801,  0.00365361, ..., -0.00576187,
          0.00987968,  0.00056312],
        [ 0.00502719,  0.0106297 , -0.00514524, ...,  0.00606945,
          0.00654817, -0.00433787],
        [ 0.00358599, -0.00257   ,  0.00474215, ..., -0.00819106,
          0.0009758 , -0.00469567],
        ...,
        [ 0.00208981,  0.01763259, -0.01958698, ...,  0.01154696,
         -0.009217  ,  0.00837894],
        [-0.0042141 , -0.00337535, -0.01742578, ..., -0.00651149,
          0.008242  ,  0.00044077],
        [-0.00331672,  0.00542616,  0.00861929, ...,  0.00442049,
         -0.00129945,  0.00199285]],

...,

[[ 0.00066472, -0.00265332, -0.00420681, ...,  0.00356282,
         -0.00272015, -0.00122229],
        [-0.00913083, -0.00096118, -0.00214067, ..., -0.00021383,
         -0.00040713, -0.0020092 ],
        [-0.0020909 ,  0.00057581, -0.00198702, ..., -0.00334667,
         -0.00107645, -0.00010455],
        ...,
        [-0.0086949 , -0.00821191,  0.00208952, ..., -0.00145943,
         -0.01484594, -0.00135062],
        [ 0.00198201,  0.000815  ,  0.00522504, ...,  0.00207165,
          0.0071472 , -0.00770016],
        [ 0.01279963,  0.00408498, -0.00312828, ...,  0.00053384,
         -0.00386579, -0.00476406]]], dtype=float32)
                },
                'query': {
                    'b': Array([[-0.05727649,  0.04552313, -0.04670092, ...,  0.00146665,
         0.02421395, -0.00091885],
       [-0.0103084 ,  0.00550128, -0.01701137, ...,  0.02177138,
         0.01786684,  0.00672262],
       [-0.19440088,  0.23860118,  0.19767429, ..., -0.2909671 ,
         0.26273695,  0.06778763],
       ...,
       [-0.01284689, -0.00560991, -0.00838155, ..., -0.00923357,
         0.00546553,  0.01274918],
       [-0.04595913,  0.00850777, -0.00376768, ...,  0.00486895,
        -0.06195865, -0.02272983],
       [-0.07012997, -0.08104558,  0.06873379, ...,  0.04658027,
        -0.02241261,  0.02544953]], dtype=float32),
                    'dora_m': Array([[[0.46142048, 0.4500186 , 0.34651643, ..., 0.31194046,
         0.3929578 , 0.32459253],
        [0.2763315 , 0.2899008 , 0.4391211 , ..., 0.49179116,
         0.39203978, 0.41947088],
        [0.3319017 , 0.80007493, 0.48443404, ..., 0.65731555,
         0.45607182, 0.6975951 ],
        ...,
        [0.6744632 , 0.6936384 , 0.5252137 , ..., 0.4535748 ,
         0.5806035 , 0.4434118 ],
        [0.71431696, 0.38224635, 0.44784164, ..., 0.5832321 ,
         0.3745505 , 0.62699455],
        [0.32467127, 0.36532265, 0.2966676 , ..., 0.35534376,
         0.2847552 , 0.33106232]]], dtype=float32),
                    'lora_a': Array([[-0.10846767],
       [ 0.29452536],
       [-0.26035994],
       ...,
       [-0.79827595],
       [ 0.7376163 ],
       [-0.08403978]], dtype=float32),
                    'lora_b': Array([[-2.1385622e-05],
       [ 2.1661906e-05],
       [ 3.5902622e-06],
       ...,
       [ 1.3643391e-05],
       [-3.5790296e-07],
       [-2.8833008e-06]], dtype=float32),
                    'w': Array([[[-1.8535994e-02,  9.5346302e-04, -1.9991446e-02, ...,
          1.0161528e-02, -1.9445473e-02,  2.0938339e-02],
        [ 4.7027450e-02, -1.9295773e-02,  5.2923034e-03, ...,
         -1.1656388e-02,  6.4211138e-02,  8.2570147e-03],
        [-7.1002268e-03,  1.9432401e-02,  3.4478415e-02, ...,
         -2.4406184e-02,  8.6701987e-03,  2.6081081e-02],
        ...,
        [ 8.1564359e-02,  9.4508991e-02,  9.0464605e-03, ...,
          1.5598202e-02, -5.1820800e-02, -2.5631530e-02],
        [ 8.2158990e-02, -1.8991157e-02, -4.1691676e-02, ...,
         -4.6170041e-02,  3.6681116e-02, -4.2611934e-02],
        [-8.7982453e-03, -5.2305171e-03,  2.0775546e-03, ...,
         -3.1847455e-02, -2.3901056e-02,  2.6486890e-02]],

...,

[[-2.9711812e-03, -3.1196219e-03,  3.0905388e-03, ...,
          5.2482258e-03, -7.5695748e-03, -9.2584523e-04],
        [-5.9177619e-03, -2.8969548e-04,  1.5838912e-03, ...,
          1.1437699e-02,  6.0875183e-03, -4.7865333e-03],
        [-2.0276653e-03,  6.7625120e-03,  6.7691165e-03, ...,
          2.1833940e-03,  4.4609946e-03,  5.0599249e-03],
        ...,
        [-7.3539508e-03, -6.0827681e-03, -1.7312258e-03, ...,
          1.2796396e-02, -9.6651735e-03,  1.4769682e-04],
        [-5.5717267e-03, -1.3548125e-03,  4.8010638e-03, ...,
         -6.7730816e-03, -5.1152851e-03, -5.0927578e-03],
        [ 2.5766164e-03,  1.0111049e-02, -4.2173183e-03, ...,
         -6.2474562e-03,  6.0498621e-03, -6.5683518e-03]]], dtype=float32)
                },
                'value': {
                    'b': Array([[ 2.0189777e-02, -1.6526524e-02,  7.5444379e-03, ...,
        -5.4049934e-03, -2.8437520e-03, -6.9945725e-03],
       [-4.1467221e-03, -8.7834820e-03,  1.3447361e-03, ...,
        -8.4275985e-03,  6.2074391e-03,  1.4883006e-02],
       [-2.9682057e-02, -4.1251830e-04, -9.7222412e-03, ...,
         8.8203693e-04, -1.3974663e-02, -6.0880790e-03],
       ...,
       [-5.1026130e-03,  2.3056350e-03, -8.9430797e-04, ...,
        -2.3210514e-03, -1.1109073e-02, -8.0586029e-03],
       [-4.9333312e-03, -7.2803735e-03, -1.4136798e-02, ...,
         3.3284265e-03,  1.0361077e-02, -2.0654943e-02],
       [ 3.9697597e-06,  6.5383087e-03,  6.8615028e-04, ...,
        -7.1053160e-03, -5.0539179e-03,  1.7412959e-02]], dtype=float32),
                    'dora_m': Array([[[0.19660895, 0.20630334, 0.19937782, ..., 0.23164397,
         0.2141681 , 0.20716642],
        [0.33267987, 0.36794344, 0.3426522 , ..., 0.41865605,
         0.32001954, 0.4054167 ],
        [0.20704515, 0.19505848, 0.2410909 , ..., 0.16695711,
         0.2178535 , 0.20635235],
        ...,
        [0.36343625, 0.40726787, 0.32639688, ..., 0.3547897 ,
         0.39589655, 0.38189554],
        [0.35044813, 0.2305814 , 0.35292068, ..., 0.36005485,
         0.29512006, 0.43130794],
        [0.19793215, 0.24579488, 0.29478118, ..., 0.27442303,
         0.28161553, 0.27517596]]], dtype=float32),
                    'lora_a': Array([[-0.10848136],
       [ 0.29452214],
       [-0.26037097],
       ...,
       [-0.79827887],
       [ 0.737614  ],
       [-0.08404165]], dtype=float32),
                    'lora_b': Array([[-1.1509576e-05],
       [ 1.8452325e-05],
       [ 8.0346508e-06],
       ...,
       [ 1.9796178e-05],
       [ 9.7198326e-06],
       [-7.2813350e-06]], dtype=float32),
                    'w': Array([[[ 1.14159631e-02, -1.59800947e-02, -3.59197892e-03, ...,
          9.23145469e-03,  1.12693738e-02,  1.20775970e-02],
        [ 2.07490451e-03,  2.38946197e-03,  1.61821153e-02, ...,
         -1.75665040e-02, -1.09817199e-02,  1.10940356e-02],
        [-1.35025522e-02,  1.15187317e-02,  1.60536468e-02, ...,
          2.25036754e-03, -3.16550490e-03, -3.02010793e-02],
        ...,
        [-3.36391851e-02, -1.05752600e-02,  1.97917111e-02, ...,
          1.07368371e-02,  9.73035581e-04,  1.68171246e-02],
        [-4.20491706e-04, -1.12381661e-02, -3.82524543e-02, ...,
         -1.18914908e-02, -4.05434705e-03, -1.41844817e-03],
        [ 6.22682087e-03, -8.74834368e-04, -2.80750962e-03, ...,
          7.22780824e-03,  2.30072369e-03, -1.35360025e-02]],

...,

[[-3.12572159e-03, -4.58090147e-03,  4.04954772e-04, ...,
         -4.26897453e-03,  1.00395828e-03,  3.13942088e-04],
        [ 6.21284498e-03,  1.91911927e-03,  6.82118209e-03, ...,
          1.77347346e-03,  3.33468284e-04, -1.69885426e-03],
        [ 6.59609400e-03, -2.01542350e-03,  5.82617614e-03, ...,
         -6.56293333e-03,  4.57923766e-03, -3.00541869e-03],
        ...,
        [ 5.23311552e-03,  5.85666252e-03,  1.00663514e-03, ...,
          4.23181895e-03, -4.45733871e-03,  5.47718396e-03],
        [ 2.15847930e-03, -2.96204607e-03,  6.17724471e-03, ...,
         -5.99337555e-03,  2.74075102e-03, -4.38916869e-03],
        [ 5.89331030e-04, -4.49438952e-03,  1.91602285e-03, ...,
          2.74458597e-03,  5.07364632e-04,  1.85802171e-03]]],      dtype=float32)
                }
            }
        },
        'x_layers_1': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': Array([ 0.00140202,  0.0150145 ,  0.01619885, ..., -0.0080475 ,
        0.00246115,  0.00246259], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.2506078, 1.3255891, 1.1087109, ..., 1.1848671, 1.0188419,
        0.9663545]], dtype=float32),
                        'lora_a': Array([[-0.10844937],
       [ 0.29455122],
       [-0.26035064],
       ...,
       [-0.7982376 ],
       [ 0.7376421 ],
       [-0.08403882]], dtype=float32),
                        'lora_b': Array([[ 6.8539694e-06],
       [ 0.0000000e+00],
       [ 7.1304553e-06],
       ...,
       [-1.0114218e-05],
       [ 1.8498655e-05],
       [-1.3987385e-05]], dtype=float32),
                        'w': Array([[-0.00787557,  0.01338205,  0.02889786, ..., -0.0941308 ,
         0.03565976, -0.01046602],
       [ 0.05106761,  0.04127442, -0.02096361, ..., -0.10027318,
        -0.02925511,  0.03009592],
       [ 0.05306583, -0.0506343 ,  0.01457519, ..., -0.09564968,
         0.04032068,  0.03279277],
       ...,
       [ 0.00158152,  0.00071211, -0.00561694, ...,  0.02137044,
         0.03315446,  0.03018216],
       [-0.04751755,  0.0064413 ,  0.02847996, ..., -0.04394938,
         0.02009094,  0.03739234],
       [ 0.05406936, -0.04232696, -0.0204008 , ...,  0.02417432,
        -0.05176357,  0.01570461]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': Array([ 0.00089951, -0.00035887, -0.00030018, ...,  0.00080614,
       -0.00512187, -0.00519201], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.0209062, 1.082342 , 1.0610949, ..., 1.0328506, 1.0591408,
        1.0562756]], dtype=float32),
                        'lora_a': Array([[-0.1084724 ],
       [ 0.29453203],
       [-0.26037055],
       ...,
       [-0.798267  ],
       [ 0.73761463],
       [-0.08404155]], dtype=float32),
                        'lora_b': Array([[ 1.85243280e-05],
       [-1.04404235e-05],
       [ 6.61929198e-06],
       ...,
       [ 1.26633431e-05],
       [ 1.45675649e-05],
       [-9.90221542e-06]], dtype=float32),
                        'w': Array([[-0.0554122 ,  0.01932358, -0.03726121, ...,  0.01832989,
         0.00719377, -0.00606497],
       [ 0.06329854,  0.01490471, -0.02579143, ...,  0.02443456,
        -0.00130729,  0.01079406],
       [ 0.0415604 ,  0.02598145, -0.04248028, ..., -0.00492932,
        -0.04967967, -0.0337037 ],
       ...,
       [-0.01317128, -0.02745201, -0.01492178, ..., -0.07122612,
        -0.0111488 ,  0.05783502],
       [ 0.03689443,  0.02630216, -0.00501244, ..., -0.00371938,
        -0.0267868 , -0.00740327],
       [-0.02904673, -0.04345071, -0.04526708, ..., -0.01844003,
        -0.00169297, -0.02380434]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': Array([ 0.00193563,  0.00100008, -0.00270318, ..., -0.01044109,
       -0.00056009,  0.00339484], dtype=float32),
                    'scale': Array([-0.0271673 , -0.02952997, -0.02053728, ..., -0.02958241,
        0.01236532, -0.00586465], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': Array([1.083301 , 1.0569358, 1.0748862, ..., 0.9615119, 1.041255 ,
       1.0239404], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': Array([[ 0.00106008,  0.00027187,  0.00024831, ..., -0.00085115,
        -0.00164053,  0.00174351],
       [-0.00210733, -0.00113148,  0.00045784, ...,  0.00069043,
        -0.00029879,  0.00441118],
       [ 0.00184593, -0.00026972,  0.00052276, ..., -0.00254204,
         0.00205951,  0.00068552],
       ...,
       [ 0.0012805 , -0.00020245,  0.0004382 , ..., -0.00219003,
         0.00275843,  0.0028203 ],
       [-0.0011356 , -0.00077106, -0.00172974, ...,  0.00030115,
        -0.0012361 , -0.00013721],
       [-0.0013096 ,  0.00173459, -0.00126198, ...,  0.00091986,
        -0.00124292, -0.00322954]], dtype=float32),
                    'dora_m': Array([[[0.38536337, 0.36554193, 0.44078323, ..., 0.38747635,
         0.40256983, 0.39221957],
        [0.37189323, 0.4900769 , 0.37673432, ..., 0.4212018 ,
         0.4217316 , 0.3739317 ],
        [0.44741547, 0.50541127, 0.47890523, ..., 0.42287713,
         0.3915702 , 0.4405926 ],
        ...,
        [0.36623392, 0.42256618, 0.43054414, ..., 0.3200549 ,
         0.32338446, 0.39520517],
        [0.4860608 , 0.44153345, 0.42090914, ..., 0.41597143,
         0.41526935, 0.45153287],
        [0.4338464 , 0.4114937 , 0.45497268, ..., 0.40209767,
         0.43825415, 0.391615  ]]], dtype=float32),
                    'lora_a': Array([[-0.10847795],
       [ 0.29452395],
       [-0.26036194],
       ...,
       [-0.7982513 ],
       [ 0.7376184 ],
       [-0.08402473]], dtype=float32),
                    'lora_b': Array([[ 1.8530738e-05],
       [ 8.1460767e-06],
       [ 1.5281736e-05],
       ...,
       [ 2.6717323e-06],
       [-2.9997761e-06],
       [-9.4125762e-06]], dtype=float32),
                    'w': Array([[[-1.80063285e-02, -5.09520667e-03,  5.54195978e-03, ...,
         -4.16026497e-03, -1.01513192e-02, -5.73090138e-03],
        [ 1.24108866e-02, -2.54758168e-03, -8.32659949e-04, ...,
          2.82166386e-03, -2.69944640e-03, -1.34731792e-02],
        [ 9.65868309e-03,  2.00738781e-03,  4.31240816e-03, ...,
         -4.11521178e-03,  6.59295451e-03,  3.14612966e-03],
        ...,
        [ 8.63340031e-03, -1.49822459e-02,  2.24748831e-02, ...,
          1.36522716e-02, -5.36048599e-03, -9.77457385e-04],
        [-2.86621023e-02,  3.08751990e-03,  1.16948429e-02, ...,
         -1.53233688e-02, -4.77650156e-03, -4.27124649e-02],
        [ 2.98779812e-02, -1.83865577e-02, -2.87840590e-02, ...,
         -9.77942161e-03, -2.34802477e-02, -2.39428822e-02]],

...,

[[ 7.48981442e-03, -1.62659085e-03, -1.02963496e-03, ...,
         -8.87682289e-03,  9.19368118e-03,  1.30671617e-02],
        [-8.24701879e-03,  2.49495194e-03,  1.91343501e-02, ...,
         -2.36123893e-02, -4.69408277e-03,  4.21702489e-03],
        [-1.06736133e-02, -3.66469217e-03, -4.78547020e-03, ...,
          5.89408027e-03,  1.24248369e-02, -9.91146808e-05],
        ...,
        [ 9.79922898e-03,  4.19031530e-05, -2.21290975e-03, ...,
          1.33938035e-02, -1.72343049e-02, -4.91101015e-03],
        [ 5.48506901e-03,  1.00362059e-02, -1.10709118e-02, ...,
         -1.10125998e-02, -9.97462124e-03,  7.39515852e-03],
        [-2.61746198e-02,  3.75667843e-03, -2.53216662e-02, ...,
          2.32830588e-02, -1.51165668e-02, -2.59454325e-02]]],      dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': Array([0.0840179 , 0.08244094, 0.06245576, 0.08689576, 0.08572489,
       0.08779805, 0.07697938, 0.09394022, 0.08097968, 0.08754763,
       0.08341486, 0.05427131, 0.08808171, 0.07731166, 0.0860716 ,
       0.07904376, 0.07062105, 0.07418054, 0.07284027, 0.07907584,
       0.06827318, 0.07395329, 0.08116236, 0.0874623 , 0.08157628,
       0.0872488 , 0.06434779, 0.06768143, 0.0991152 , 0.07790367,
       0.0773618 , 0.07666542, 0.07887019, 0.08864841, 0.05583115,
       0.09895247, 0.10453542, 0.12288661, 0.11304845, 0.1039682 ,
       0.06304675, 0.10898725, 0.1052065 , 0.07074126, 0.07930253,
       0.09781079, 0.12111677, 0.09354454, 0.08050405, 0.09708615,
       0.09194972, 0.08502522, 0.07665642, 0.09167423, 0.05380047,
       0.07951675, 0.10532993, 0.0631919 , 0.0652303 , 0.09186361,
       0.0613084 , 0.09283226, 0.10487343, 0.11428737, 0.05996763,
       0.07991926, 0.07705473, 0.08530728, 0.0627922 , 0.10198504,
       0.08804259, 0.0772656 , 0.12166466, 0.07498417, 0.0551027 ,
       0.07845861, 0.0941859 , 0.08346103, 0.06084656, 0.0929875 ],      dtype=float32)
                },
                'post': {
                    'b': Array([-8.5084306e-05, -1.1306409e-03, -3.8026206e-04, ...,
        8.4014353e-04, -2.9504241e-03, -2.7141783e-03], dtype=float32),
                    'dora_m': Array([[[0.2615482 , 0.25563622, 0.2439947 , ..., 0.27743977,
         0.22440323, 0.28170407],
        [0.25765923, 0.2939706 , 0.27973562, ..., 0.28255165,
         0.30106005, 0.28556922],
        [0.30175266, 0.26688752, 0.2832422 , ..., 0.28372487,
         0.29387408, 0.28594834],
        ...,
        [0.29269975, 0.30184668, 0.31280023, ..., 0.27423418,
         0.2876577 , 0.31824026],
        [0.28017336, 0.255455  , 0.31681693, ..., 0.26993114,
         0.28596452, 0.28776297],
        [0.29380643, 0.3101701 , 0.29397225, ..., 0.3012885 ,
         0.3036872 , 0.29270852]]], dtype=float32),
                    'lora_a': Array([[-0.10847852],
       [ 0.294541  ],
       [-0.26038304],
       ...,
       [-0.79825604],
       [ 0.73763335],
       [-0.08402013]], dtype=float32),
                    'lora_b': Array([[ 2.0177655e-05],
       [ 2.0681546e-05],
       [-2.0637648e-05],
       ...,
       [-2.0866762e-05],
       [-2.2007569e-06],
       [ 1.8250683e-05]], dtype=float32),
                    'w': Array([[[-0.00708895, -0.01480758,  0.01334359, ..., -0.01726026,
          0.00060817, -0.00494491],
        [ 0.01220738,  0.00202692, -0.00111894, ..., -0.00664444,
          0.0240082 ,  0.00878899],
        [ 0.00194961, -0.01813553,  0.00442683, ..., -0.02262122,
         -0.01098372,  0.00288465],
        ...,
        [-0.004893  , -0.00926619, -0.00032269, ...,  0.00613354,
          0.01687936,  0.00583418],
        [ 0.00492367,  0.00383521, -0.00515759, ...,  0.00415317,
          0.00398865, -0.00464302],
        [ 0.00809234,  0.00042906,  0.01076343, ...,  0.00149677,
          0.00985494, -0.00063159]],

...,

[[ 0.00828316, -0.00220981,  0.00527866, ...,  0.00705342,
          0.01052925, -0.00860018],
        [ 0.00089691, -0.00227484, -0.00601018, ..., -0.00764392,
         -0.00332279, -0.00458629],
        [ 0.00742584, -0.00383384,  0.00629323, ...,  0.00774965,
         -0.00140316, -0.00998758],
        ...,
        [-0.01038045,  0.0074235 , -0.00076588, ...,  0.00299953,
          0.01173578, -0.00073879],
        [-0.00396501, -0.00384969, -0.00263106, ...,  0.0007746 ,
         -0.00322374,  0.00686969],
        [-0.00165319, -0.00069949, -0.00341704, ...,  0.00016102,
          0.00203316, -0.00223182]]], dtype=float32)
                },
                'query': {
                    'b': Array([[ 0.01952027,  0.02592703,  0.01507241, ...,  0.01695433,
        -0.00907723,  0.00838097],
       [-0.01057168, -0.00455518, -0.01358485, ...,  0.00340533,
         0.01626122, -0.01695308],
       [-0.00089808,  0.00056304, -0.02251254, ...,  0.0155376 ,
        -0.01954361,  0.0131553 ],
       ...,
       [ 0.0013416 ,  0.00240092,  0.00654386, ...,  0.02299946,
        -0.01865708, -0.00998419],
       [ 0.02600852, -0.0133097 , -0.0117526 , ...,  0.02061324,
         0.03217793,  0.0197717 ],
       [-0.00274182,  0.01528802, -0.01193944, ...,  0.02231876,
        -0.01095807, -0.01857167]], dtype=float32),
                    'dora_m': Array([[[0.47948712, 0.5028795 , 0.52071905, ..., 0.45554927,
         0.6400639 , 0.52903026],
        [0.5286403 , 0.48682663, 0.46731868, ..., 0.56363755,
         0.52869946, 0.5251107 ],
        [0.4675773 , 0.5442484 , 0.5856209 , ..., 0.559333  ,
         0.6385352 , 0.5633995 ],
        ...,
        [0.45784378, 0.44041935, 0.41728064, ..., 0.4418479 ,
         0.46732882, 0.41660374],
        [0.65318453, 0.4952946 , 0.54076195, ..., 0.5277434 ,
         0.62166375, 0.53507954],
        [0.5234772 , 0.6732376 , 0.62285554, ..., 0.5897458 ,
         0.52376735, 0.532405  ]]], dtype=float32),
                    'lora_a': Array([[-0.10845468],
       [ 0.29454905],
       [-0.26035517],
       ...,
       [-0.79824513],
       [ 0.7376287 ],
       [-0.08402891]], dtype=float32),
                    'lora_b': Array([[-1.3783408e-05],
       [-1.6565022e-05],
       [ 3.2628327e-06],
       ...,
       [ 5.2546925e-06],
       [ 3.3100555e-06],
       [ 4.2820698e-06]], dtype=float32),
                    'w': Array([[[-0.00167942,  0.01055444, -0.01093264, ..., -0.02210693,
          0.01847492, -0.00255125],
        [ 0.01091587, -0.00381827,  0.01870508, ..., -0.00639131,
         -0.0212311 , -0.002378  ],
        [ 0.01231089, -0.00437073, -0.01020596, ...,  0.05093126,
          0.01480385,  0.01470868],
        ...,
        [ 0.00866831,  0.01642605,  0.01773182, ...,  0.00787777,
          0.00646162,  0.0289838 ],
        [ 0.03137889,  0.01285845,  0.01455711, ..., -0.02560751,
          0.02575761, -0.00764195],
        [-0.01348505, -0.04771884, -0.00423431, ...,  0.00435013,
          0.0177009 , -0.0179799 ]],

...,

[[ 0.01646333,  0.00423958,  0.01651123, ..., -0.01476896,
          0.006426  ,  0.022834  ],
        [-0.00171471,  0.01116127,  0.00777352, ..., -0.0167604 ,
         -0.00763599, -0.01560871],
        [-0.00409038, -0.000114  , -0.02664903, ..., -0.0039424 ,
          0.01968246, -0.01381156],
        ...,
        [ 0.01327849, -0.03327335,  0.01283411, ...,  0.02662477,
         -0.01906407, -0.00113409],
        [ 0.02000252, -0.02245389, -0.00270883, ...,  0.01630181,
         -0.01032738,  0.01874355],
        [ 0.00188271, -0.002353  , -0.05625219, ...,  0.00055419,
         -0.04353327, -0.0223032 ]]], dtype=float32)
                },
                'value': {
                    'b': Array([[-1.4370600e-03, -1.2042513e-03,  1.0407623e-03, ...,
         1.4411649e-03, -8.5053267e-04,  7.9507163e-06],
       [-8.6065737e-04, -2.0831903e-03,  2.4297652e-03, ...,
         1.1107546e-03,  9.7443461e-03,  1.0844832e-04],
       [ 2.4037904e-03, -1.5152757e-03, -1.1029590e-05, ...,
         2.4529195e-03,  2.2916619e-03,  3.8734211e-03],
       ...,
       [ 4.2531057e-03, -1.1110497e-03,  4.8523545e-03, ...,
        -5.3573218e-03, -1.9030088e-03, -7.4371318e-03],
       [ 5.7160575e-03, -2.3649796e-03,  2.0968895e-03, ...,
         5.3650416e-03,  7.2402111e-03,  1.4226218e-03],
       [-8.1616519e-03,  4.6124598e-03, -8.5368557e-03, ...,
         7.2550117e-03,  9.2172315e-03, -8.0767851e-03]], dtype=float32),
                    'dora_m': Array([[[0.3088568 , 0.2970335 , 0.28107744, ..., 0.32049185,
         0.25769407, 0.3036742 ],
        [0.275908  , 0.3507761 , 0.3304603 , ..., 0.33387133,
         0.33066568, 0.32940993],
        [0.3397523 , 0.30376956, 0.30107582, ..., 0.30771422,
         0.32612264, 0.34057006],
        ...,
        [0.32837015, 0.34558183, 0.383396  , ..., 0.28547767,
         0.3064036 , 0.3629731 ],
        [0.33432278, 0.29297066, 0.37051696, ..., 0.2962845 ,
         0.3201727 , 0.31476173],
        [0.32278425, 0.32515886, 0.35435697, ..., 0.33256602,
         0.3680951 , 0.33803067]]], dtype=float32),
                    'lora_a': Array([[-0.1084739 ],
       [ 0.29454657],
       [-0.26036003],
       ...,
       [-0.79823774],
       [ 0.7376553 ],
       [-0.08404048]], dtype=float32),
                    'lora_b': Array([[ 1.0517151e-05],
       [ 1.0368214e-05],
       [-2.0456351e-05],
       ...,
       [ 5.6098639e-07],
       [-6.5533545e-06],
       [-1.1938995e-06]], dtype=float32),
                    'w': Array([[[ 0.0024759 , -0.00276686,  0.0023287 , ..., -0.00156066,
         -0.00649969,  0.00501549],
        [ 0.00389328,  0.00884204, -0.00255053, ..., -0.0137577 ,
         -0.01047342,  0.00884087],
        [-0.01649504,  0.00430972,  0.00274404, ...,  0.02594932,
         -0.04495201, -0.01399373],
        ...,
        [ 0.00888101, -0.00785696,  0.00229611, ...,  0.00703543,
         -0.007503  ,  0.00540859],
        [ 0.0085597 , -0.01634004,  0.00304436, ..., -0.00409839,
         -0.00071542,  0.00278309],
        [-0.00284988, -0.00032437, -0.01045181, ...,  0.01348813,
         -0.02370471,  0.00539708]],

...,

[[ 0.00473459, -0.00507944,  0.00289661, ...,  0.00038999,
         -0.0081128 , -0.00403884],
        [-0.00585744, -0.00309845,  0.00449292, ...,  0.00646797,
          0.00730119,  0.00418908],
        [ 0.01212458, -0.00163699, -0.00837895, ..., -0.00508494,
          0.00191359, -0.01044772],
        ...,
        [-0.00874585,  0.01692164,  0.00437238, ...,  0.01866874,
          0.00768705,  0.01460969],
        [-0.00084818, -0.00683727, -0.00774377, ..., -0.00031492,
         -0.0020496 ,  0.00169231],
        [ 0.00199953,  0.01445957,  0.00079529, ...,  0.00466131,
         -0.00959731,  0.00840501]]], dtype=float32)
                }
            }
        },
        'x_layers_10': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': Array([-0.0067778 , -0.01533606, -0.00765442, ...,  0.0045179 ,
       -0.00582889, -0.01096233], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.0983046, 1.1869583, 1.0642086, ..., 1.2278987, 1.3065704,
        1.1312832]], dtype=float32),
                        'lora_a': Array([[-0.10848615],
       [ 0.29455504],
       [-0.26037169],
       ...,
       [-0.7982789 ],
       [ 0.73765224],
       [-0.08400372]], dtype=float32),
                        'lora_b': Array([[ 1.4919997e-06],
       [ 9.8503097e-06],
       [ 4.1351041e-06],
       ...,
       [-8.7940762e-06],
       [ 1.2203855e-05],
       [ 1.0096498e-05]], dtype=float32),
                        'w': Array([[-0.03108777,  0.01383287, -0.01179256, ...,  0.00302204,
        -0.01438387, -0.01885613],
       [ 0.00535768, -0.04698981, -0.03603473, ..., -0.01139537,
        -0.06621313,  0.03310591],
       [ 0.04872718, -0.0184944 , -0.04604091, ..., -0.04631934,
        -0.02415592,  0.05293693],
       ...,
       [-0.00610507, -0.02534259, -0.02141761, ..., -0.01412116,
         0.00220612, -0.03082991],
       [-0.06125214,  0.01565345,  0.03463744, ...,  0.00639873,
         0.05287894, -0.03431875],
       [-0.01022425,  0.05079507, -0.00049477, ...,  0.01767915,
        -0.00905383,  0.06914392]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': Array([ 0.00332026,  0.00228718,  0.00081935, ...,  0.00143483,
       -0.00280289,  0.0016605 ], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.1923897, 1.4533943, 1.3592458, ..., 1.3414602, 1.3559332,
        1.6517143]], dtype=float32),
                        'lora_a': Array([[-0.10848703],
       [ 0.2945362 ],
       [-0.26038966],
       ...,
       [-0.79826576],
       [ 0.7376283 ],
       [-0.08402517]], dtype=float32),
                        'lora_b': Array([[-1.07033622e-06],
       [-1.09151715e-05],
       [ 7.55222857e-07],
       ...,
       [-1.50665273e-05],
       [ 8.79217623e-06],
       [ 2.45582669e-06]], dtype=float32),
                        'w': Array([[-0.01038639, -0.00440558, -0.01641116, ..., -0.05271889,
         0.00743694, -0.00781685],
       [-0.06110938, -0.03921739, -0.03602229, ...,  0.00572572,
         0.00305964,  0.0342207 ],
       [-0.02868134, -0.01991718,  0.02981003, ...,  0.02833821,
        -0.03532288,  0.02274652],
       ...,
       [-0.04388466, -0.04275953, -0.06564004, ...,  0.03234898,
         0.05879903, -0.04443722],
       [ 0.06963926, -0.01727262,  0.04664012, ..., -0.03569457,
        -0.00283924, -0.00034458],
       [ 0.00845499, -0.02555194, -0.02030128, ...,  0.06764334,
         0.03920732,  0.06198909]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': Array([-0.00652681,  0.00998919,  0.01091337, ..., -0.00884033,
       -0.01000154,  0.00719102], dtype=float32),
                    'scale': Array([ 0.00018895, -0.0016679 , -0.00290126, ..., -0.00919969,
       -0.00058326,  0.00534971], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': Array([1.1182874, 1.1435981, 1.1135991, ..., 1.1002538, 1.118429 ,
       1.1571561], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': Array([[-0.00028102, -0.00059104, -0.00017435, ..., -0.00119082,
        -0.00010263, -0.00071618],
       [ 0.00225487, -0.00275545,  0.00204913, ..., -0.00047768,
         0.00016692,  0.00026425],
       [ 0.00023483,  0.00177506, -0.00131258, ...,  0.0014215 ,
        -0.0040562 ,  0.00237817],
       ...,
       [ 0.00106694,  0.0002561 ,  0.00144923, ...,  0.0013506 ,
         0.00120613, -0.00084387],
       [ 0.00105687, -0.00163147, -0.00046463, ...,  0.00094481,
         0.00178576,  0.00060445],
       [ 0.00122252, -0.00197313,  0.00088491, ...,  0.00058077,
         0.00029764, -0.00264683]], dtype=float32),
                    'dora_m': Array([[[0.5430324 , 0.55198115, 0.53475755, ..., 0.5254069 ,
         0.6540496 , 0.5536293 ],
        [0.5027069 , 0.59888583, 0.56136924, ..., 0.5595747 ,
         0.566561  , 0.40334854],
        [0.52357805, 0.56374204, 0.43854976, ..., 0.67809826,
         0.5153692 , 0.44705185],
        ...,
        [0.53408116, 0.49903566, 0.53142464, ..., 0.5059704 ,
         0.53414536, 0.50800174],
        [0.44073448, 0.58092   , 0.49840945, ..., 0.49162456,
         0.5281699 , 0.39432272],
        [0.43779507, 0.51193464, 0.5869895 , ..., 0.5098001 ,
         0.542407  , 0.6022885 ]]], dtype=float32),
                    'lora_a': Array([[-0.10847314],
       [ 0.29452476],
       [-0.26036423],
       ...,
       [-0.79825526],
       [ 0.73762167],
       [-0.08402664]], dtype=float32),
                    'lora_b': Array([[-5.6186750e-06],
       [ 2.8835848e-06],
       [ 8.8307042e-06],
       ...,
       [ 1.1999508e-05],
       [ 2.0275675e-07],
       [ 1.2588047e-05]], dtype=float32),
                    'w': Array([[[-1.22559890e-02, -8.39809235e-03, -1.48427067e-02, ...,
          6.22703508e-03,  1.86309591e-02,  3.24096270e-02],
        [-2.01482568e-02,  8.23511835e-03, -3.59878410e-03, ...,
          5.64934453e-03,  8.32179096e-03,  1.41915437e-02],
        [-2.01054364e-02, -6.93156384e-03, -9.31501039e-04, ...,
         -4.10634056e-02,  6.05812436e-03,  1.22190723e-02],
        ...,
        [-4.57775360e-03,  1.78224165e-02, -1.38880024e-02, ...,
         -1.27791585e-02,  1.98074467e-02, -3.26664071e-03],
        [ 4.63145878e-03,  7.87381578e-05, -2.41322033e-02, ...,
         -1.23561732e-02,  6.57504192e-03,  6.37360057e-03],
        [ 4.68460284e-03,  4.65884060e-03, -1.86497867e-02, ...,
          3.24863405e-03,  1.32136820e-02,  5.25168935e-03]],

...,

[[ 1.13208451e-04,  3.52317952e-02,  4.93009761e-03, ...,
          1.20255938e-02,  5.44467615e-03,  1.82285514e-02],
        [ 1.27479937e-02, -1.24411648e-02,  1.48131000e-02, ...,
         -2.89926236e-03,  4.92918771e-03,  9.24422313e-03],
        [ 2.80147996e-02,  2.45938282e-02, -4.92357509e-03, ...,
         -5.41405566e-03, -1.25533426e-02,  2.44517159e-02],
        ...,
        [ 2.46858466e-02, -2.21450329e-02, -1.72019340e-02, ...,
          2.06563505e-03,  1.24617934e-03,  1.31983301e-02],
        [ 6.06509508e-04, -6.05794648e-03,  2.30345670e-02, ...,
         -7.03589944e-03, -1.05391983e-02, -1.45047577e-02],
        [-7.19427271e-03, -2.63856980e-03,  6.21147733e-03, ...,
         -2.99184263e-04, -2.09438652e-02,  6.05939981e-03]]],      dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': Array([0.2826307 , 0.29122394, 0.2804533 , 0.27769658, 0.2847127 ,
       0.24761622, 0.2710216 , 0.3152247 , 0.2765104 , 0.2484727 ,
       0.2533266 , 0.27915084, 0.24555965, 0.3073783 , 0.2655005 ,
       0.21189591, 0.35232496, 0.30432513, 0.30837816, 0.25320613,
       0.29300117, 0.25749105, 0.30691883, 0.2562598 , 0.31728888,
       0.26139805, 0.24298659, 0.29966286, 0.24946421, 0.28548405,
       0.21169291, 0.26257178, 0.31407136, 0.2906975 , 0.27783373,
       0.26280415, 0.2737751 , 0.30103633, 0.2973439 , 0.25732696,
       0.27588543, 0.2568364 , 0.3071738 , 0.24696194, 0.29834172,
       0.23746881, 0.27979898, 0.23967174, 0.25484917, 0.31370804,
       0.24649821, 0.2343228 , 0.24406944, 0.30672845, 0.29227123,
       0.24194668, 0.24848543, 0.27132946, 0.25255135, 0.24560043,
       0.28916624, 0.29397386, 0.2424356 , 0.27622077, 0.30238086,
       0.34377092, 0.2524194 , 0.26836666, 0.27820814, 0.24207515,
       0.27608994, 0.27313074, 0.29991195, 0.2935175 , 0.23672445,
       0.28691012, 0.24444716, 0.24724962, 0.2769544 , 0.20427427],      dtype=float32)
                },
                'post': {
                    'b': Array([ 7.9627399e-04,  6.5674139e-03,  5.6891823e-03, ...,
        4.3108783e-05, -3.6937576e-03,  2.3444982e-03], dtype=float32),
                    'dora_m': Array([[[0.42438787, 0.42514828, 0.41871065, ..., 0.4401789 ,
         0.45613766, 0.43351287],
        [0.4071341 , 0.3904299 , 0.40059063, ..., 0.38920042,
         0.37596035, 0.43680158],
        [0.43352613, 0.43641424, 0.4243551 , ..., 0.4318674 ,
         0.41704458, 0.4515158 ],
        ...,
        [0.4294712 , 0.44587347, 0.41890606, ..., 0.43028352,
         0.433481  , 0.4277656 ],
        [0.38639587, 0.399807  , 0.39171946, ..., 0.39806756,
         0.36692393, 0.40915114],
        [0.4356283 , 0.4051668 , 0.4282043 , ..., 0.43957967,
         0.4277489 , 0.4368648 ]]], dtype=float32),
                    'lora_a': Array([[-0.10848585],
       [ 0.29455256],
       [-0.26038882],
       ...,
       [-0.7982715 ],
       [ 0.73765314],
       [-0.08400862]], dtype=float32),
                    'lora_b': Array([[ 1.2020925e-05],
       [-8.9048999e-06],
       [-1.9190875e-05],
       ...,
       [ 7.7846272e-07],
       [-9.3114004e-06],
       [-1.1753739e-06]], dtype=float32),
                    'w': Array([[[ 0.00146442,  0.01960428, -0.00436989, ..., -0.0122232 ,
          0.00920983, -0.00245794],
        [-0.0095711 ,  0.00907172, -0.0116214 , ..., -0.00354761,
         -0.00579774,  0.00126938],
        [ 0.00276704,  0.00060504, -0.00706686, ..., -0.00294393,
          0.00962784,  0.01267469],
        ...,
        [ 0.00700635, -0.02775826,  0.00443999, ...,  0.01711492,
         -0.021339  ,  0.00733781],
        [-0.0081798 ,  0.01323767,  0.01253227, ..., -0.01629487,
          0.0106214 ,  0.02292852],
        [ 0.00385438,  0.0126046 ,  0.00792869, ...,  0.01123373,
         -0.00201993, -0.00607193]],

...,

[[-0.00086033, -0.02251566,  0.0042542 , ..., -0.00318098,
          0.00196227, -0.00998912],
        [-0.00541877,  0.00864972, -0.00619451, ..., -0.02292626,
         -0.00532148,  0.01265658],
        [ 0.00966086,  0.00343994, -0.00895778, ...,  0.01743233,
          0.00556804,  0.03041199],
        ...,
        [-0.00209922, -0.00457141,  0.00087416, ...,  0.00012216,
         -0.00576033,  0.01509899],
        [ 0.00219567, -0.01285906,  0.00986449, ...,  0.00465276,
          0.00352433, -0.02275107],
        [-0.00331315,  0.00438775, -0.00717978, ..., -0.00826449,
         -0.00581287, -0.002987  ]]], dtype=float32)
                },
                'query': {
                    'b': Array([[ 0.01600958,  0.00398826,  0.00773052, ..., -0.01548967,
         0.00642794,  0.01298024],
       [ 0.00263645, -0.00473251, -0.00969235, ...,  0.00956154,
        -0.00312625, -0.02954184],
       [-0.00979827, -0.00529572,  0.0223179 , ...,  0.00354051,
         0.00967805, -0.00740001],
       ...,
       [ 0.00558155,  0.0006147 ,  0.00570557, ...,  0.01541192,
        -0.00349259, -0.00577878],
       [ 0.03326915,  0.02490069, -0.03056262, ...,  0.0211593 ,
         0.02024012,  0.03402796],
       [-0.02150407, -0.00563736,  0.01245746, ...,  0.00610306,
        -0.01175688, -0.00551602]], dtype=float32),
                    'dora_m': Array([[[0.84210396, 0.81192124, 0.8938123 , ..., 0.93823737,
         0.75542253, 0.8918173 ],
        [0.7302126 , 0.6351719 , 0.8881558 , ..., 0.6471459 ,
         0.7104286 , 0.77921534],
        [0.89461595, 0.77993846, 0.86100703, ..., 0.6571361 ,
         0.8543439 , 0.8843835 ],
        ...,
        [0.9594508 , 0.8704423 , 0.90137845, ..., 0.7800498 ,
         0.83896345, 0.9940558 ],
        [0.55472803, 0.62694544, 0.6206565 , ..., 0.6579522 ,
         0.68475395, 0.52248   ],
        [0.798847  , 0.68206865, 0.7338591 , ..., 0.7870018 ,
         0.7195704 , 0.8200386 ]]], dtype=float32),
                    'lora_a': Array([[-0.1084858 ],
       [ 0.294553  ],
       [-0.26038754],
       ...,
       [-0.79827714],
       [ 0.7376201 ],
       [-0.08402611]], dtype=float32),
                    'lora_b': Array([[ 1.0456335e-05],
       [ 1.5322668e-05],
       [ 1.9051939e-05],
       ...,
       [ 7.5706403e-06],
       [ 1.2870032e-06],
       [-1.4513865e-05]], dtype=float32),
                    'w': Array([[[ 1.40356151e-02,  5.94120426e-03, -2.29084790e-02, ...,
         -5.35522103e-02, -1.32354945e-02,  1.50852511e-02],
        [ 1.75219439e-02, -6.20160834e-04, -4.98878174e-02, ...,
          2.22302740e-03,  4.82971693e-04, -3.72398198e-02],
        [ 2.58184019e-02,  1.29804909e-02, -1.26741165e-02, ...,
          1.46833034e-02, -1.93886776e-02, -2.34460309e-02],
        ...,
        [-3.15314792e-02, -8.34475458e-02, -4.39461842e-02, ...,
         -2.68571340e-02,  1.01708723e-02, -3.54705937e-02],
        [ 8.31054058e-03,  3.87277617e-03, -2.14894675e-02, ...,
         -1.33718094e-02, -2.90948711e-02,  2.14557368e-02],
        [-1.25645357e-03,  2.59165093e-02,  2.21974850e-02, ...,
          2.54525691e-02, -1.30615011e-02, -5.37769049e-02]],

...,

[[-1.60638113e-02, -6.68555452e-03,  1.93832051e-02, ...,
         -3.06526269e-03, -1.15770791e-02,  3.39199603e-03],
        [ 1.71611663e-02,  2.29309052e-02,  4.75622788e-02, ...,
          6.47983840e-03, -8.60652328e-03,  1.79007335e-03],
        [ 4.43298332e-02,  4.19816077e-02, -2.22983640e-02, ...,
         -2.87520867e-02, -2.16403510e-02,  6.30730949e-03],
        ...,
        [-3.26161981e-02,  2.95277941e-03, -1.02727003e-02, ...,
          1.27500594e-02,  1.43683655e-02, -2.82541923e-02],
        [-6.02143863e-03, -4.94311610e-03, -2.88585536e-02, ...,
         -3.91545370e-02, -2.86819078e-02, -9.28332284e-03],
        [-9.95085947e-03,  1.78441207e-03,  2.01135557e-02, ...,
          1.29047967e-03,  7.24823726e-03, -7.70971971e-03]]],      dtype=float32)
                },
                'value': {
                    'b': Array([[ 4.5682947e-04,  1.1954156e-04, -3.4605192e-03, ...,
        -2.7432817e-03, -6.2608662e-05, -2.2155052e-04],
       [ 3.4376332e-03,  4.1965075e-05,  2.1032067e-05, ...,
         1.5058093e-03, -1.0985093e-03,  3.8899151e-03],
       [ 7.2707783e-04, -6.1844854e-04,  1.3926377e-03, ...,
         3.1390032e-03, -6.2617409e-04,  2.1017287e-03],
       ...,
       [ 1.3336381e-03, -2.3447198e-03,  1.5440906e-03, ...,
         9.3644871e-05,  2.7158058e-03, -3.3990832e-03],
       [ 3.1390661e-04, -3.3300222e-04,  6.3425634e-04, ...,
        -6.3325587e-04, -9.4877614e-04, -6.7050627e-04],
       [ 2.8241735e-03, -7.6000334e-04, -5.7119469e-04, ...,
        -1.2217826e-03, -2.1154580e-04,  6.1394530e-04]], dtype=float32),
                    'dora_m': Array([[[0.46888867, 0.458271  , 0.4767595 , ..., 0.46525612,
         0.48104453, 0.4889108 ],
        [0.4757418 , 0.45145696, 0.4568846 , ..., 0.45461684,
         0.41964948, 0.48806554],
        [0.5037011 , 0.47038314, 0.4542899 , ..., 0.48451313,
         0.45562622, 0.48059458],
        ...,
        [0.46242195, 0.48711476, 0.47487006, ..., 0.46234247,
         0.483294  , 0.4765595 ],
        [0.4192615 , 0.45170122, 0.44531947, ..., 0.4454639 ,
         0.40857166, 0.46339822],
        [0.45751932, 0.4541766 , 0.47715   , ..., 0.48058715,
         0.465252  , 0.47489518]]], dtype=float32),
                    'lora_a': Array([[-0.10849117],
       [ 0.294555  ],
       [-0.26039213],
       ...,
       [-0.79827917],
       [ 0.73761475],
       [-0.0840391 ]], dtype=float32),
                    'lora_b': Array([[ 1.5462481e-05],
       [-1.7367925e-05],
       [-4.8553220e-06],
       ...,
       [ 1.5290658e-05],
       [ 1.9064755e-05],
       [ 7.8209187e-06]], dtype=float32),
                    'w': Array([[[ 1.72701497e-02, -6.67018292e-04, -8.94597825e-03, ...,
         -3.28017096e-03, -4.27454263e-02,  6.90392451e-03],
        [-2.00283024e-02, -2.27912087e-02,  3.64550166e-02, ...,
          1.12843597e-02, -7.17920298e-03,  3.81224486e-03],
        [-1.83604751e-02, -5.59525704e-03,  1.36792324e-02, ...,
          1.07971877e-02,  1.05734896e-02,  5.53275226e-04],
        ...,
        [ 1.50924991e-03,  1.53016811e-02,  7.70701421e-03, ...,
          2.38751504e-03,  3.14462073e-02,  2.86122821e-02],
        [-1.59093644e-02,  1.95494387e-02, -4.26613865e-03, ...,
          4.92176507e-04,  1.68949869e-02, -1.51739400e-02],
        [ 6.87260740e-03, -5.25096944e-03,  9.77834454e-04, ...,
         -9.87444259e-03,  2.22718250e-03,  1.64193325e-02]],

...,

[[ 6.84548286e-04,  2.69682538e-02, -1.54047934e-02, ...,
          1.53855504e-02,  8.06880649e-03,  4.89617931e-03],
        [-1.54312905e-02, -6.75051892e-03,  1.15704141e-03, ...,
          6.85672089e-03,  1.08955754e-02, -1.00796996e-02],
        [-1.09128449e-02, -1.59259364e-02,  2.11769976e-02, ...,
          1.58200483e-03,  1.07090431e-03, -1.44510455e-02],
        ...,
        [-1.52886715e-02,  1.66227240e-02,  7.04474747e-03, ...,
          2.01132130e-02,  4.22550086e-03,  6.48447312e-03],
        [ 4.47970722e-03,  5.69607876e-03,  1.09875211e-02, ...,
          3.46971527e-02,  5.85914729e-03,  2.69552274e-03],
        [-1.18758455e-02,  1.96109060e-04, -4.45304206e-03, ...,
         -1.07747018e-02,  6.75870990e-03,  9.27476212e-03]]],      dtype=float32)
                }
            }
        },
        'x_layers_11': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': Array([-0.00175118, -0.00260024, -0.07228219, ..., -0.00627134,
        0.00413736, -0.00720975], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[0.9200448, 1.0077602, 2.0899606, ..., 1.1732534, 1.3468015,
        1.1367917]], dtype=float32),
                        'lora_a': Array([[-0.10844861],
       [ 0.29455465],
       [-0.2603505 ],
       ...,
       [-0.7982785 ],
       [ 0.73764515],
       [-0.08403609]], dtype=float32),
                        'lora_b': Array([[-2.0879765e-05],
       [ 5.2943972e-07],
       [ 0.0000000e+00],
       ...,
       [ 1.1435607e-05],
       [-9.0093163e-06],
       [-1.9819950e-05]], dtype=float32),
                        'w': Array([[ 0.02213248, -0.00037442,  0.06455456, ..., -0.00868579,
         0.0104142 , -0.02710419],
       [ 0.03270084,  0.0241459 , -0.04848697, ..., -0.03151397,
         0.03347551,  0.00065035],
       [-0.00683347,  0.02788812, -0.01665919, ...,  0.0314375 ,
         0.02865025, -0.0341912 ],
       ...,
       [ 0.01531753,  0.02818437,  0.05320358, ...,  0.03597291,
        -0.00072661,  0.04358203],
       [ 0.02493588,  0.03074985,  0.03032211, ...,  0.00291781,
         0.00067245, -0.0406046 ],
       [-0.03414572, -0.01924999, -0.00704158, ..., -0.00375799,
         0.03458655,  0.01684766]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': Array([ 0.00636536,  0.00142258, -0.00197367, ...,  0.00075185,
       -0.00083699,  0.00319561], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.2713882, 1.5184681, 1.3088295, ..., 1.4292513, 1.3142416,
        1.6887299]], dtype=float32),
                        'lora_a': Array([[-0.10844827],
       [ 0.2945542 ],
       [-0.26037082],
       ...,
       [-0.7982574 ],
       [ 0.7376434 ],
       [-0.08400069]], dtype=float32),
                        'lora_b': Array([[-1.7034687e-05],
       [-1.7558152e-05],
       [-1.2884102e-05],
       ...,
       [-1.6990583e-05],
       [ 2.0267909e-05],
       [ 1.6294985e-05]], dtype=float32),
                        'w': Array([[ 0.00576453,  0.01392252, -0.01244743, ...,  0.02634548,
        -0.0390147 ,  0.05118013],
       [-0.04683452, -0.02094531, -0.05402165, ...,  0.02874108,
         0.01810433,  0.01536882],
       [-0.08094323, -0.17057261,  0.06222852, ...,  0.00614128,
         0.06856798, -0.02509728],
       ...,
       [-0.02571372,  0.11391928, -0.05397959, ..., -0.00348219,
         0.01380937, -0.06948423],
       [ 0.03470301, -0.04579668, -0.00139861, ..., -0.05194421,
        -0.05121345,  0.08306472],
       [ 0.01261504, -0.02599149,  0.00764601, ...,  0.0212124 ,
        -0.00760618,  0.00859129]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': Array([-9.9775520e-05,  1.3316037e-02,  1.2384055e-02, ...,
       -9.1059729e-03, -2.5081664e-02, -1.0536548e-02], dtype=float32),
                    'scale': Array([ 0.00314306,  0.00437152,  0.00764915, ...,  0.02828045,
        0.01542678, -0.02259208], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': Array([1.156839 , 1.1383926, 1.1070566, ..., 1.1095787, 1.1333247,
       1.1252159], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': Array([[ 6.2043295e-04, -1.3959286e-03,  2.3537264e-03, ...,
         2.6737602e-04,  4.6616062e-03,  3.2361986e-03],
       [-2.2848645e-04, -1.9312546e-03, -7.2261330e-04, ...,
         3.8449914e-04,  2.2252691e-03, -3.6747850e-04],
       [-4.6333583e-04,  7.0525962e-04,  9.5361407e-04, ...,
        -2.3124523e-03, -1.8506297e-03,  4.0299914e-04],
       ...,
       [-8.8660774e-05, -1.8478214e-04, -3.6815304e-04, ...,
        -1.2149196e-03, -1.4899155e-03,  7.4451091e-05],
       [ 2.6595125e-03,  3.3997109e-03, -3.8310296e-03, ...,
         3.4757920e-03,  1.5175877e-03, -5.4881223e-03],
       [-6.1853061e-05, -3.0119701e-03,  1.3912504e-03, ...,
         3.1533630e-03,  8.3954731e-04, -2.0716980e-03]], dtype=float32),
                    'dora_m': Array([[[0.65482837, 0.60560185, 0.51018035, ..., 0.6010325 ,
         0.5692064 , 0.49387988],
        [0.48169833, 0.5297338 , 0.6453005 , ..., 0.544811  ,
         0.53890115, 0.5651028 ],
        [0.4625506 , 0.5766735 , 0.5416806 , ..., 0.6030242 ,
         0.4871151 , 0.54194427],
        ...,
        [0.5159496 , 0.5405723 , 0.5726131 , ..., 0.5705121 ,
         0.5544942 , 0.48639446],
        [0.5126148 , 0.6237152 , 0.4415264 , ..., 0.47120613,
         0.540262  , 0.4658818 ],
        [0.5394921 , 0.49724182, 0.62283015, ..., 0.484452  ,
         0.49090564, 0.5999742 ]]], dtype=float32),
                    'lora_a': Array([[-0.10848814],
       [ 0.29452   ],
       [-0.26036128],
       ...,
       [-0.798274  ],
       [ 0.7376318 ],
       [-0.08403021]], dtype=float32),
                    'lora_b': Array([[-9.3448161e-06],
       [ 9.0124331e-06],
       [ 2.1618655e-06],
       ...,
       [ 4.2034176e-06],
       [-1.4373490e-05],
       [-5.3665140e-06]], dtype=float32),
                    'w': Array([[[-0.00457982,  0.0079292 ,  0.01626714, ..., -0.02177358,
         -0.00753616,  0.0111543 ],
        [ 0.00736282, -0.01522866, -0.03390208, ...,  0.01205219,
          0.00368838, -0.02313531],
        [-0.00222201,  0.01081776, -0.0121273 , ...,  0.00954761,
          0.00297511,  0.02277266],
        ...,
        [-0.01193842,  0.02725654, -0.01004553, ..., -0.00874436,
         -0.01074831, -0.00778359],
        [ 0.01702505, -0.001029  , -0.01283343, ..., -0.01121063,
         -0.00362994, -0.00895869],
        [ 0.01208098, -0.01486417,  0.00574359, ..., -0.00670334,
         -0.01481716,  0.0051486 ]],

...,

[[ 0.01201486, -0.0106921 , -0.01455065, ...,  0.0024754 ,
          0.01651331, -0.00582526],
        [ 0.00676857, -0.0060736 , -0.00843585, ..., -0.00529255,
          0.03072165, -0.00851097],
        [-0.00565433,  0.03521806, -0.0024964 , ..., -0.0134829 ,
          0.00173979,  0.01382122],
        ...,
        [ 0.0012847 ,  0.01398057, -0.01192073, ..., -0.00321589,
          0.00887635,  0.00224419],
        [ 0.00562963, -0.02689695, -0.00206297, ..., -0.01880339,
          0.00894797, -0.0019288 ],
        [ 0.01913709, -0.01316302, -0.03764893, ..., -0.0134073 ,
          0.02239419, -0.05215774]]], dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': Array([0.3156545 , 0.27060285, 0.25025174, 0.34601673, 0.30242926,
       0.26294008, 0.31233415, 0.3209547 , 0.2849913 , 0.3288459 ,
       0.2951874 , 0.28546813, 0.34476817, 0.33740777, 0.284232  ,
       0.28655267, 0.26494822, 0.29337803, 0.30076003, 0.2882022 ,
       0.30189607, 0.271989  , 0.3013342 , 0.30964464, 0.26838246,
       0.2603131 , 0.32541406, 0.32179737, 0.35538927, 0.273313  ,
       0.3013977 , 0.2551599 , 0.29802316, 0.297576  , 0.27485722,
       0.3644293 , 0.29362065, 0.27971974, 0.32013175, 0.3102587 ,
       0.29194638, 0.26629296, 0.32570788, 0.32607472, 0.2795367 ,
       0.3140577 , 0.2892701 , 0.31884027, 0.30475014, 0.26978436,
       0.28342175, 0.31857198, 0.28328526, 0.29521295, 0.3464261 ,
       0.30882302, 0.29280803, 0.28246257, 0.28882617, 0.3388947 ,
       0.25662103, 0.3308743 , 0.32114318, 0.29870152, 0.30939123,
       0.28852645, 0.30617124, 0.2908906 , 0.30798033, 0.25875768,
       0.25241807, 0.33816648, 0.3175192 , 0.2886452 , 0.30905277,
       0.32988924, 0.3363946 , 0.30680224, 0.34127322, 0.32526475],      dtype=float32)
                },
                'post': {
                    'b': Array([ 5.6089791e-03,  8.0252551e-03,  5.5846493e-03, ...,
       -1.1894177e-03, -7.2966306e-03, -2.8434333e-05], dtype=float32),
                    'dora_m': Array([[[0.43908253, 0.46729374, 0.44159955, ..., 0.4543115 ,
         0.4397317 , 0.46240062],
        [0.39667442, 0.42408815, 0.41745678, ..., 0.42899603,
         0.45052087, 0.42545107],
        [0.4533764 , 0.4312953 , 0.39767206, ..., 0.43023437,
         0.4395852 , 0.4247762 ],
        ...,
        [0.4381729 , 0.40769988, 0.41493493, ..., 0.430935  ,
         0.4460243 , 0.4456532 ],
        [0.44408885, 0.46779153, 0.4488128 , ..., 0.45319396,
         0.4471334 , 0.43733212],
        [0.4597264 , 0.43994263, 0.40696415, ..., 0.42206278,
         0.44003183, 0.43369192]]], dtype=float32),
                    'lora_a': Array([[-0.10848624],
       [ 0.29455057],
       [-0.26038858],
       ...,
       [-0.79827535],
       [ 0.73765516],
       [-0.08400498]], dtype=float32),
                    'lora_b': Array([[ 1.2560981e-05],
       [-1.9147958e-05],
       [-1.4902183e-05],
       ...,
       [ 1.4783218e-05],
       [-1.7752058e-05],
       [-1.3450427e-06]], dtype=float32),
                    'w': Array([[[ 0.00616197, -0.01140843, -0.01220697, ...,  0.03211016,
         -0.00915002, -0.00320862],
        [-0.00163407, -0.00726635,  0.0026372 , ...,  0.01594584,
         -0.01256238, -0.00934431],
        [ 0.00269712, -0.0087405 , -0.00707724, ..., -0.01187651,
         -0.00985631,  0.01202096],
        ...,
        [-0.00177817, -0.0155329 ,  0.00039959, ..., -0.00509618,
          0.01596226, -0.0090281 ],
        [-0.00362233, -0.00166358,  0.010353  , ...,  0.01115924,
         -0.00344578, -0.01445935],
        [-0.00719647, -0.00679773,  0.00159126, ...,  0.00904022,
          0.00913885, -0.01345026]],

...,

[[ 0.00819326, -0.00378347, -0.00536378, ..., -0.00384975,
          0.00924117,  0.00283117],
        [ 0.00150048, -0.0014319 , -0.00483689, ...,  0.01057441,
          0.01947576,  0.00472962],
        [-0.0212515 ,  0.01883786, -0.00505871, ...,  0.0143664 ,
          0.00042881,  0.01159056],
        ...,
        [ 0.0004472 ,  0.01151392,  0.0013491 , ...,  0.00855052,
         -0.00520287, -0.01697323],
        [ 0.01939233, -0.00611647,  0.01255679, ...,  0.02077544,
         -0.01566506,  0.00383031],
        [-0.00312147, -0.01002034, -0.00377799, ...,  0.00377198,
         -0.00486637,  0.00745471]]], dtype=float32)
                },
                'query': {
                    'b': Array([[-0.01902875, -0.00392321, -0.01692069, ...,  0.01277379,
        -0.02017537, -0.01966368],
       [-0.00514293,  0.01013522,  0.00640971, ...,  0.00700698,
        -0.00208709,  0.0016113 ],
       [-0.01669493,  0.00687944,  0.00167014, ...,  0.00559093,
        -0.01526949, -0.01213987],
       ...,
       [ 0.01259618, -0.017484  ,  0.00524346, ...,  0.00086777,
         0.01313006,  0.01849369],
       [ 0.00576792,  0.00946417,  0.00764743, ..., -0.01395615,
        -0.00582852,  0.00378093],
       [-0.00593368,  0.00607188, -0.002292  , ..., -0.01927363,
        -0.00423207,  0.00327838]], dtype=float32),
                    'dora_m': Array([[[0.7672689 , 0.71676123, 0.7789787 , ..., 0.7998458 ,
         0.8349001 , 0.81385857],
        [0.7626534 , 0.68991125, 0.6342171 , ..., 0.75669324,
         0.7934009 , 0.713425  ],
        [0.8279263 , 0.6869622 , 0.6840315 , ..., 0.60322857,
         0.86074436, 0.7828992 ],
        ...,
        [0.7701897 , 0.86873066, 0.6815448 , ..., 0.7676876 ,
         0.80533475, 0.7294045 ],
        [0.74893665, 0.9255758 , 0.81932825, ..., 0.95134497,
         0.97493845, 0.9233438 ],
        [0.6700128 , 0.6605961 , 0.5263657 , ..., 0.80462784,
         0.6880284 , 0.74881995]]], dtype=float32),
                    'lora_a': Array([[-0.10845528],
       [ 0.2945536 ],
       [-0.26036328],
       ...,
       [-0.79827815],
       [ 0.73762107],
       [-0.08404076]], dtype=float32),
                    'lora_b': Array([[ 7.1116638e-06],
       [-5.1602269e-06],
       [-1.8787636e-05],
       ...,
       [ 7.9858064e-06],
       [ 2.0493844e-05],
       [-1.7685143e-06]], dtype=float32),
                    'w': Array([[[ 6.7107915e-03,  5.2108735e-02, -3.5726141e-02, ...,
          1.0018836e-02, -6.6985086e-02, -2.9997937e-02],
        [-4.8798919e-02, -3.0955179e-02, -4.5889658e-03, ...,
         -1.4168783e-02, -1.9515565e-02, -4.5938934e-03],
        [ 1.7948883e-02, -1.9678289e-02, -2.7322422e-03, ...,
          3.4518955e-05,  1.0276827e-02,  3.0288305e-02],
        ...,
        [ 3.2520901e-02,  3.1548835e-02,  3.2453798e-02, ...,
          3.0249842e-02, -1.3171470e-02, -3.0540736e-02],
        [-2.0047123e-02,  2.0765377e-02,  1.7037734e-02, ...,
         -3.1289443e-02,  3.2366186e-02,  2.4516651e-04],
        [-1.4139613e-03, -2.7877068e-02,  2.9665206e-03, ...,
         -8.7918406e-03, -3.1671613e-03, -6.0802251e-03]],

...,

[[ 3.9989837e-02, -8.2005756e-03,  1.7767716e-02, ...,
          1.8468142e-03,  2.7596813e-03,  1.0810228e-02],
        [-3.4049381e-02,  1.8112855e-02,  4.7151122e-02, ...,
          2.6763078e-02, -3.0164555e-02, -1.5285182e-02],
        [ 2.9293558e-02,  1.1467175e-02, -1.0890862e-02, ...,
         -2.2795664e-03,  2.0938506e-02,  1.5154293e-02],
        ...,
        [-5.0077680e-02,  2.2626992e-02, -8.6043058e-03, ...,
          2.3196422e-02, -2.4148957e-03, -1.6109640e-02],
        [-5.3558229e-03, -7.1678106e-03, -9.8695131e-03, ...,
         -1.5392502e-02,  1.8007027e-03, -2.0988617e-02],
        [-1.3041359e-02, -4.1281167e-02, -1.1608910e-02, ...,
         -1.1827341e-02, -7.6007438e-03, -2.9945277e-02]]], dtype=float32)
                },
                'value': {
                    'b': Array([[ 2.1569493e-04,  1.6116437e-03, -9.6050935e-04, ...,
        -6.5116951e-04, -6.0316140e-04,  4.0845378e-04],
       [ 2.7031007e-03,  2.5382696e-03,  2.3867956e-03, ...,
         3.8557494e-04,  1.9661232e-04,  1.8881332e-03],
       [ 7.4628938e-04,  2.0475746e-03, -1.8672572e-03, ...,
        -9.9990261e-04,  1.9131867e-03, -4.5626456e-04],
       ...,
       [-7.5348897e-04, -1.4569069e-03, -1.7060187e-04, ...,
        -2.4431029e-03, -4.1715769e-04,  4.0554241e-03],
       [ 2.6874433e-03,  2.4147295e-05, -3.1609035e-03, ...,
        -2.1084265e-03, -1.5388979e-03, -1.8308936e-03],
       [ 2.6015663e-03, -2.3570405e-05, -1.0428695e-03, ...,
        -1.4327377e-03,  3.6195872e-04,  5.5156567e-04]], dtype=float32),
                    'dora_m': Array([[[0.47741705, 0.4798827 , 0.48175925, ..., 0.50169927,
         0.48669475, 0.49802363],
        [0.43795538, 0.46319747, 0.46544752, ..., 0.47173735,
         0.47798076, 0.4757697 ],
        [0.5026454 , 0.46952355, 0.44015247, ..., 0.47697455,
         0.48747164, 0.47433925],
        ...,
        [0.4848705 , 0.48102722, 0.47703362, ..., 0.46659756,
         0.4744314 , 0.50504553],
        [0.49576706, 0.5180444 , 0.48605308, ..., 0.4751922 ,
         0.48454028, 0.48085985],
        [0.49372834, 0.48930952, 0.4521509 , ..., 0.46853775,
         0.48437396, 0.5011597 ]]], dtype=float32),
                    'lora_a': Array([[-0.10845193],
       [ 0.2945551 ],
       [-0.2603576 ],
       ...,
       [-0.79827905],
       [ 0.7376182 ],
       [-0.08404135]], dtype=float32),
                    'lora_b': Array([[-9.8184955e-06],
       [ 1.8910439e-05],
       [ 5.3088743e-06],
       ...,
       [-2.1354357e-05],
       [-1.3592518e-05],
       [ 8.9837740e-06]], dtype=float32),
                    'w': Array([[[ 6.31503412e-04,  1.45085622e-03,  3.17231473e-03, ...,
         -3.25035416e-02,  2.26579071e-03, -1.32843880e-02],
        [-1.35540571e-02, -1.90303975e-03,  2.11610203e-03, ...,
         -2.68081836e-02,  1.45731904e-02,  2.98355296e-02],
        [-2.29272507e-02, -6.38898881e-03, -5.79964509e-03, ...,
          5.89815527e-03, -1.21480143e-02, -5.48514305e-04],
        ...,
        [ 2.93679964e-02,  1.65273417e-02, -7.91169144e-03, ...,
          3.39419208e-03,  5.04504656e-03,  4.24112082e-02],
        [ 5.78944013e-03,  4.27541137e-03, -2.87316274e-03, ...,
          7.80965155e-03, -1.54245207e-02,  3.92249711e-02],
        [ 9.51418560e-03, -6.88549364e-03,  1.10508539e-02, ...,
         -2.77678296e-02,  1.95166506e-02,  8.34298786e-03]],

...,

[[-2.66917534e-02, -2.11770367e-02,  1.28460834e-02, ...,
          2.05245968e-02,  8.59414972e-03, -1.65698007e-02],
        [-4.99683758e-03,  4.65656910e-03,  1.11284396e-02, ...,
          1.69438235e-02,  2.64371349e-03, -1.35477455e-02],
        [ 1.36140212e-02, -6.97145518e-03, -1.70006659e-02, ...,
          1.33774430e-02, -4.16422449e-03, -4.53103613e-03],
        ...,
        [ 1.50281899e-02,  2.04638466e-02, -5.65202953e-03, ...,
         -2.92194309e-04,  1.09987482e-02,  2.95356428e-03],
        [ 3.92625947e-03, -1.02107776e-02, -3.78664173e-02, ...,
          4.30914294e-03,  9.04566702e-03,  1.14939539e-02],
        [ 2.49127694e-03,  1.19736474e-02,  8.10928084e-03, ...,
         -1.89916193e-02,  1.08265905e-02,  1.38882212e-02]]],      dtype=float32)
                }
            }
        },
        'x_layers_12': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': Array([-0.005293  , -0.00186352, -0.00976755, ..., -0.00339723,
       -0.00447999,  0.00120996], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.0571557 , 1.0269492 , 1.0352547 , ..., 1.2856908 , 1.0439367 ,
        0.92964906]], dtype=float32),
                        'lora_a': Array([[-0.10845279],
       [ 0.29455474],
       [-0.2603497 ],
       ...,
       [-0.798274  ],
       [ 0.7376495 ],
       [-0.0840397 ]], dtype=float32),
                        'lora_b': Array([[ 1.4822314e-05],
       [-1.5509097e-05],
       [-8.1723392e-06],
       ...,
       [ 1.7430350e-05],
       [ 2.1478827e-05],
       [ 5.9088438e-06]], dtype=float32),
                        'w': Array([[-0.05177403,  0.03388924,  0.000139  , ...,  0.05519148,
         0.00972583, -0.02497132],
       [ 0.00069568,  0.0167134 , -0.02542666, ..., -0.063407  ,
         0.02492764, -0.00733338],
       [ 0.03444348,  0.00616424, -0.05731964, ..., -0.05109103,
        -0.04094215, -0.01131166],
       ...,
       [-0.02862806, -0.01231155, -0.01399679, ...,  0.05987118,
        -0.02321418,  0.04145474],
       [-0.00463293, -0.02615651,  0.01964143, ...,  0.06114722,
        -0.00820913,  0.02821478],
       [-0.0261556 ,  0.00931396, -0.02824642, ...,  0.03728772,
        -0.07729811, -0.00626508]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': Array([ 0.00122742,  0.00010827,  0.00017512, ..., -0.00082751,
       -0.00092059,  0.00197941], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.2721658, 1.3986415, 1.2732021, ..., 1.3964716, 1.2481251,
        1.6003611]], dtype=float32),
                        'lora_a': Array([[-0.10848017],
       [ 0.29454264],
       [-0.26034972],
       ...,
       [-0.7982601 ],
       [ 0.73765177],
       [-0.08399902]], dtype=float32),
                        'lora_b': Array([[ 1.6365273e-05],
       [-2.0880370e-05],
       [-5.8661931e-06],
       ...,
       [-1.4282488e-05],
       [ 1.4401434e-05],
       [-7.5435196e-06]], dtype=float32),
                        'w': Array([[ 0.02870242,  0.01056961, -0.05915664, ...,  0.04323499,
        -0.00249853, -0.02161617],
       [ 0.01951083,  0.00519961,  0.00644839, ...,  0.03814481,
         0.01063481, -0.02081017],
       [ 0.01685014,  0.01569146, -0.01697563, ...,  0.01234141,
        -0.03345832,  0.04551524],
       ...,
       [ 0.04950652,  0.01376844, -0.00967372, ...,  0.06082512,
         0.09064892,  0.03400097],
       [-0.00100831, -0.02080236,  0.03347975, ...,  0.04448443,
         0.0319938 ,  0.01907505],
       [ 0.03527055, -0.01972844,  0.01550603, ...,  0.01602129,
         0.00324922, -0.01287165]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': Array([-0.00601908,  0.02212059,  0.02046599, ..., -0.00412445,
       -0.01955507, -0.00602043], dtype=float32),
                    'scale': Array([0.0143517 , 0.00214911, 0.01108167, ..., 0.01265304, 0.03338145,
       0.0163041 ], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': Array([1.1374506, 1.1442024, 1.0783095, ..., 1.1171334, 1.1079882,
       1.1521581], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': Array([[-1.8685563e-04, -5.9532432e-04, -2.4054792e-04, ...,
         6.2968215e-04,  5.7730218e-04, -4.4597613e-04],
       [-2.2711651e-03,  2.0482612e-03, -2.2182227e-03, ...,
        -5.3416757e-04, -1.0647925e-03, -1.2073171e-03],
       [ 4.0595946e-03,  2.3135310e-03, -2.3099906e-03, ...,
         1.4927713e-03,  1.5450851e-03, -1.6390900e-04],
       ...,
       [ 2.7009632e-04,  5.9131434e-04,  1.3146417e-03, ...,
         8.6348550e-04,  1.2941496e-04,  1.5455941e-04],
       [-7.8688434e-04,  2.6643097e-03,  2.3450158e-04, ...,
         1.3286179e-03,  3.6279860e-04,  1.4461885e-03],
       [-6.0682447e-04,  1.9236694e-03,  2.6605760e-03, ...,
        -2.7408556e-03,  3.2813117e-05,  1.4253191e-03]], dtype=float32),
                    'dora_m': Array([[[0.66907036, 0.46850607, 0.5471381 , ..., 0.6616052 ,
         0.5656944 , 0.64162   ],
        [0.48782867, 0.5663539 , 0.42218995, ..., 0.58823025,
         0.5461479 , 0.56289923],
        [0.5304807 , 0.730262  , 0.5391924 , ..., 0.5227691 ,
         0.74766034, 0.59703463],
        ...,
        [0.6021005 , 0.51070833, 0.43162796, ..., 0.5246829 ,
         0.57143027, 0.5469597 ],
        [0.5600127 , 0.67428076, 0.6106244 , ..., 0.5856159 ,
         0.55979574, 0.6510917 ],
        [0.58434844, 0.56614375, 0.5600536 , ..., 0.5837473 ,
         0.51727694, 0.5558782 ]]], dtype=float32),
                    'lora_a': Array([[-0.10847027],
       [ 0.29452127],
       [-0.26035568],
       ...,
       [-0.79826856],
       [ 0.73765534],
       [-0.08400052]], dtype=float32),
                    'lora_b': Array([[ 1.36560175e-05],
       [ 9.16819226e-06],
       [ 1.44594208e-06],
       ...,
       [-1.92179396e-05],
       [-5.32242780e-07],
       [-1.94927870e-05]], dtype=float32),
                    'w': Array([[[-0.04404075,  0.0221399 ,  0.00467264, ...,  0.0320016 ,
          0.00014809, -0.00875474],
        [ 0.00923844, -0.0026586 ,  0.02337929, ..., -0.00236505,
          0.0094295 ,  0.00125714],
        [-0.04657682, -0.02400855, -0.01212377, ...,  0.01054189,
         -0.05642375, -0.00059953],
        ...,
        [ 0.03438158,  0.00055848, -0.00796749, ...,  0.02653286,
         -0.0355892 ,  0.01229315],
        [ 0.00275324,  0.0128751 , -0.02055732, ..., -0.00229726,
         -0.00360587,  0.0258247 ],
        [-0.01309112,  0.02698477,  0.04702034, ..., -0.00881093,
         -0.01423186, -0.04045918]],

...,

[[ 0.01742528, -0.00615192,  0.00217556, ..., -0.01009956,
          0.03087114, -0.02774436],
        [-0.02156098, -0.01062487,  0.00157245, ...,  0.00781925,
         -0.00496119, -0.0196279 ],
        [ 0.00954189,  0.0165382 ,  0.00890495, ..., -0.01508511,
         -0.00338188,  0.02362835],
        ...,
        [-0.00717626,  0.01471218,  0.00573334, ...,  0.00746967,
          0.00471901, -0.00187142],
        [-0.01044797,  0.03700609, -0.02869983, ..., -0.01165732,
         -0.00662046,  0.00494565],
        [-0.00374042, -0.00148811, -0.02519495, ...,  0.0100678 ,
         -0.00519629, -0.00252824]]], dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': Array([0.35639364, 0.34132004, 0.3678308 , 0.29332113, 0.3548022 ,
       0.3196629 , 0.34824505, 0.28022683, 0.34007522, 0.35805207,
       0.3447933 , 0.36301646, 0.3236222 , 0.32413065, 0.33023334,
       0.28743845, 0.34377715, 0.3393906 , 0.33953124, 0.32523715,
       0.3169872 , 0.31388932, 0.38340393, 0.36926383, 0.34337118,
       0.31441355, 0.34962794, 0.3661602 , 0.31924438, 0.35972518,
       0.33061033, 0.29423946, 0.31563804, 0.34502977, 0.35413533,
       0.37948734, 0.36898082, 0.29570147, 0.37050843, 0.3258704 ,
       0.37426075, 0.3357056 , 0.27940238, 0.3129776 , 0.38641983,
       0.3623211 , 0.34729213, 0.34133965, 0.32058895, 0.34806046,
       0.36713976, 0.3717784 , 0.37632713, 0.3323835 , 0.38991728,
       0.3203513 , 0.35657376, 0.33545214, 0.35677052, 0.36801088,
       0.32906908, 0.2758519 , 0.33993274, 0.30850875, 0.3776776 ,
       0.32257536, 0.34567377, 0.33076546, 0.36784562, 0.3375391 ,
       0.3420226 , 0.33942178, 0.35138342, 0.33556384, 0.36306417,
       0.31934866, 0.37257168, 0.39450267, 0.32862386, 0.3617384 ],      dtype=float32)
                },
                'post': {
                    'b': Array([-0.00214796,  0.00832791,  0.01047556, ..., -0.00122844,
       -0.00554719, -0.00057771], dtype=float32),
                    'dora_m': Array([[[0.47342858, 0.43475717, 0.46136633, ..., 0.44729346,
         0.43084082, 0.44730216],
        [0.4645725 , 0.45870584, 0.45704758, ..., 0.4515763 ,
         0.44421583, 0.43038422],
        [0.43916816, 0.4456193 , 0.44463384, ..., 0.43174604,
         0.4710136 , 0.45860824],
        ...,
        [0.43583968, 0.41243795, 0.44063637, ..., 0.440316  ,
         0.4403551 , 0.41640294],
        [0.44896275, 0.44048357, 0.45069867, ..., 0.44877183,
         0.4692447 , 0.43414825],
        [0.3835265 , 0.3787568 , 0.43085817, ..., 0.4305252 ,
         0.3892411 , 0.45414722]]], dtype=float32),
                    'lora_a': Array([[-0.10849009],
       [ 0.29455194],
       [-0.2603812 ],
       ...,
       [-0.79827505],
       [ 0.7376429 ],
       [-0.08400504]], dtype=float32),
                    'lora_b': Array([[-2.0025967e-05],
       [ 5.8457031e-06],
       [-1.7065882e-05],
       ...,
       [-7.4818631e-06],
       [-4.2321649e-06],
       [-1.8663073e-05]], dtype=float32),
                    'w': Array([[[ 0.02869009,  0.00943764,  0.00531797, ..., -0.01740457,
          0.01775924,  0.00723355],
        [-0.00965818,  0.01898344,  0.00488442, ..., -0.00729101,
         -0.01743222,  0.00118863],
        [-0.01206351, -0.00496965,  0.00367228, ...,  0.0010526 ,
          0.00596434, -0.00036469],
        ...,
        [ 0.01421978,  0.01114153, -0.0083301 , ...,  0.00824197,
         -0.00754816, -0.01727102],
        [ 0.00025025,  0.00437635,  0.00784304, ...,  0.01374844,
         -0.0200068 , -0.01633633],
        [-0.00375637, -0.00395306, -0.0253886 , ..., -0.00194593,
          0.01819008,  0.00381707]],

...,

[[ 0.00562998,  0.00938226,  0.00687099, ..., -0.00116021,
         -0.00542865,  0.00782787],
        [-0.00352626, -0.0139083 , -0.00634241, ..., -0.00363757,
          0.01029849, -0.00430407],
        [-0.00223516,  0.00650535, -0.02042501, ..., -0.0076793 ,
          0.00563121, -0.00039084],
        ...,
        [ 0.00772774, -0.00045712,  0.01417555, ...,  0.0134108 ,
         -0.01336487, -0.01822393],
        [ 0.00024862, -0.00368071, -0.00101053, ..., -0.0114745 ,
         -0.01521763,  0.02290301],
        [ 0.01499688, -0.02343859,  0.02381371, ...,  0.01503901,
          0.00494653,  0.01106588]]], dtype=float32)
                },
                'query': {
                    'b': Array([[-4.92933625e-03, -3.36020924e-02, -1.64103545e-02, ...,
         2.68765818e-03, -1.06418999e-02,  1.27139585e-02],
       [-5.93224121e-03,  1.45464456e-02, -1.70037597e-02, ...,
        -3.68339213e-04, -1.01805395e-02,  5.22182696e-03],
       [ 5.34146652e-03, -3.20420414e-03, -1.42281828e-03, ...,
         1.20814871e-02, -7.36192043e-04, -1.14789819e-02],
       ...,
       [ 2.92606885e-03,  1.45223234e-02, -3.49404365e-02, ...,
        -2.47589555e-02, -2.33482718e-02,  1.60419587e-02],
       [ 3.34996567e-03,  6.13400573e-03, -7.76892819e-04, ...,
         7.41102593e-03,  1.08059179e-02,  4.80904151e-03],
       [-1.40242314e-03,  5.81266219e-03,  5.59961842e-03, ...,
         4.84774867e-03,  6.56471308e-03,  3.65962405e-05]],      dtype=float32),
                    'dora_m': Array([[[0.7034931 , 0.8449176 , 0.85103023, ..., 0.73617494,
         0.62314814, 0.79978955],
        [0.8693337 , 0.9489377 , 0.7642453 , ..., 0.9026105 ,
         0.9188802 , 0.8452059 ],
        [0.7752524 , 0.6781312 , 0.8158559 , ..., 0.8827212 ,
         0.6725244 , 0.7757164 ],
        ...,
        [0.66894513, 0.8214849 , 0.73996186, ..., 0.8679951 ,
         0.8052258 , 0.64019006],
        [0.75135267, 0.790864  , 0.9153559 , ..., 0.72891724,
         0.81347233, 0.8262301 ],
        [0.82730705, 0.7542209 , 0.8202622 , ..., 0.8430626 ,
         0.8416618 , 0.8915491 ]]], dtype=float32),
                    'lora_a': Array([[-0.10847769],
       [ 0.29455364],
       [-0.26035103],
       ...,
       [-0.7982768 ],
       [ 0.73762375],
       [-0.08403516]], dtype=float32),
                    'lora_b': Array([[1.4240426e-05],
       [2.0973112e-05],
       [1.6316539e-05],
       ...,
       [2.0102898e-05],
       [1.1638186e-05],
       [1.7366474e-05]], dtype=float32),
                    'w': Array([[[ 0.00874558,  0.01253717, -0.0166835 , ..., -0.00588645,
         -0.01074735,  0.04966896],
        [ 0.0316819 ,  0.01123518, -0.04319673, ..., -0.06064183,
          0.00224262,  0.01926875],
        [ 0.00822638,  0.01188794, -0.01699981, ..., -0.03090369,
         -0.0030938 , -0.0177431 ],
        ...,
        [-0.02056288,  0.00035961, -0.02575049, ..., -0.06145574,
         -0.00127565,  0.01225445],
        [ 0.03344519, -0.01105584,  0.00055006, ..., -0.05779998,
         -0.01317679, -0.00418735],
        [ 0.04997033,  0.03178057,  0.00088188, ..., -0.02633874,
          0.01553694,  0.03623275]],

...,

[[-0.03653891,  0.00268992,  0.00891523, ...,  0.02581696,
         -0.00660179,  0.02126133],
        [ 0.02626838,  0.01582966,  0.02101961, ...,  0.03602818,
         -0.02656418, -0.04796021],
        [ 0.00724903, -0.01323678,  0.01695194, ...,  0.03835869,
         -0.00470296,  0.03168933],
        ...,
        [ 0.00065492, -0.02142965,  0.00766696, ..., -0.01565139,
          0.00737353, -0.00363003],
        [-0.01310692, -0.01751375,  0.03384865, ..., -0.011557  ,
         -0.02706607, -0.0287535 ],
        [ 0.01559199,  0.0444673 , -0.03096942, ..., -0.00501271,
         -0.04978823, -0.03677486]]], dtype=float32)
                },
                'value': {
                    'b': Array([[-1.7844223e-04, -4.9078255e-03,  8.7939674e-04, ...,
        -1.0633421e-03,  4.2153242e-05,  8.6045424e-05],
       [ 3.3815745e-03, -3.3321648e-04,  2.6374005e-03, ...,
        -7.3969213e-04,  2.4472391e-03, -9.2108030e-04],
       [-1.4806272e-03, -7.8831380e-04, -3.1488092e-04, ...,
        -2.8324886e-03,  2.2519149e-03,  2.4730733e-03],
       ...,
       [ 1.5006368e-03,  1.1138931e-03,  1.2836054e-03, ...,
        -1.9483248e-03,  1.9670355e-03,  2.0739788e-03],
       [-1.1379643e-03,  3.1626232e-03, -2.6456336e-03, ...,
         6.7569410e-05, -6.4134382e-04,  7.1093894e-04],
       [ 2.6015060e-03,  2.1737472e-03, -6.5869543e-05, ...,
        -4.1303635e-03, -3.1118204e-03,  3.3902212e-03]], dtype=float32),
                    'dora_m': Array([[[0.50447375, 0.46724018, 0.5114046 , ..., 0.47381523,
         0.4477036 , 0.47463435],
        [0.479018  , 0.48179698, 0.4771351 , ..., 0.45694238,
         0.46394122, 0.45775568],
        [0.48131165, 0.5033466 , 0.4691962 , ..., 0.45853248,
         0.493827  , 0.49630377],
        ...,
        [0.47209585, 0.4712924 , 0.47317794, ..., 0.48263744,
         0.46746635, 0.4520368 ],
        [0.47884083, 0.49162316, 0.47913992, ..., 0.48660073,
         0.5433875 , 0.4614753 ],
        [0.42755646, 0.40069628, 0.4653801 , ..., 0.47444674,
         0.42853773, 0.50395155]]], dtype=float32),
                    'lora_a': Array([[-0.10849037],
       [ 0.29455492],
       [-0.26034984],
       ...,
       [-0.79827815],
       [ 0.7376159 ],
       [-0.08404032]], dtype=float32),
                    'lora_b': Array([[-3.7683114e-06],
       [-1.3667470e-05],
       [-1.7201050e-05],
       ...,
       [ 1.9934032e-05],
       [-9.6611475e-06],
       [-1.6273601e-05]], dtype=float32),
                    'w': Array([[[-2.03157421e-02, -9.15903971e-03, -1.30444383e-02, ...,
          7.36718590e-04, -4.22519026e-03,  4.15073941e-03],
        [ 6.77502574e-03,  7.28660147e-04, -2.45866347e-02, ...,
         -9.14037600e-03,  8.25326890e-03, -8.17570277e-03],
        [-1.19243674e-02, -5.63951815e-03, -1.43306993e-03, ...,
          6.03869790e-03,  2.69319844e-02,  7.01575167e-03],
        ...,
        [-1.51856628e-03,  1.58385616e-02,  1.25889126e-02, ...,
         -4.13889857e-03, -2.59260560e-05,  9.66171175e-03],
        [-1.85218062e-02,  1.11629032e-02, -1.86361023e-03, ...,
          2.39219842e-03,  2.28052828e-02,  1.11672943e-02],
        [ 6.10933779e-03, -8.68937373e-03, -6.37223851e-03, ...,
          6.03761245e-03, -8.80723353e-03, -1.75700113e-02]],

...,

[[-4.68146466e-02, -9.26453341e-03, -2.99443770e-02, ...,
          1.04604037e-02, -1.59611069e-02, -2.64821807e-03],
        [-1.03302562e-04, -6.50553359e-03, -4.10049642e-03, ...,
          2.07577683e-02,  1.37467496e-02, -7.90071487e-03],
        [ 2.25197431e-03, -1.07433237e-02,  5.16068377e-03, ...,
          2.34127864e-02, -1.93827450e-02,  3.20600392e-03],
        ...,
        [ 2.07280535e-02, -7.11975561e-04, -1.49935167e-02, ...,
          6.89100660e-03, -4.35881875e-03, -5.95880672e-03],
        [ 1.53170358e-02, -1.15952250e-02, -1.87348202e-02, ...,
         -2.00649239e-02,  1.02016167e-03, -6.05729036e-03],
        [-7.36127701e-03,  1.23309800e-02,  4.21048189e-03, ...,
         -2.26222374e-03,  1.46400416e-03, -1.72078945e-02]]],      dtype=float32)
                }
            }
        },
        'x_layers_13': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': Array([-0.0034478 , -0.0628284 , -0.00682131, ..., -0.01855724,
       -0.00795195, -0.01122809], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.1060199, 2.4845655, 1.186707 , ..., 1.2604355, 1.120649 ,
        1.2176853]], dtype=float32),
                        'lora_a': Array([[-0.10844901],
       [ 0.2945548 ],
       [-0.2603502 ],
       ...,
       [-0.7982789 ],
       [ 0.73761916],
       [-0.08401955]], dtype=float32),
                        'lora_b': Array([[-7.2031744e-06],
       [ 0.0000000e+00],
       [ 1.8693202e-06],
       ...,
       [-2.3444059e-06],
       [-5.0362482e-06],
       [ 0.0000000e+00]], dtype=float32),
                        'w': Array([[-0.03151614, -0.0080314 , -0.04668951, ...,  0.01049254,
         0.03591558,  0.00079862],
       [-0.06258383, -0.04206429,  0.02148353, ...,  0.00677137,
         0.03879655, -0.02585962],
       [-0.03223332, -0.04780781, -0.02068976, ..., -0.01317758,
         0.01165592, -0.0696369 ],
       ...,
       [ 0.01789408,  0.01969726,  0.04556585, ...,  0.06506349,
         0.03052077,  0.06799204],
       [-0.01471116,  0.04009691,  0.03642584, ...,  0.02687883,
         0.00796544, -0.02481566],
       [ 0.04239961,  0.01176694,  0.04996835, ..., -0.05828528,
         0.02227721,  0.02466226]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': Array([ 1.7072462e-03, -8.4020954e-05,  3.6142219e-03, ...,
       -1.6132628e-03,  6.7724548e-05,  2.5193540e-03], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.2484009, 1.3118861, 1.1905327, ..., 1.3474647, 1.262712 ,
        1.5220765]], dtype=float32),
                        'lora_a': Array([[-0.10846844],
       [ 0.29453942],
       [-0.2603705 ],
       ...,
       [-0.7982612 ],
       [ 0.73763996],
       [-0.08401774]], dtype=float32),
                        'lora_b': Array([[-1.7862851e-05],
       [-3.1506936e-06],
       [-1.1678742e-05],
       ...,
       [ 1.9711313e-05],
       [ 2.1225369e-05],
       [-2.1498881e-05]], dtype=float32),
                        'w': Array([[ 0.01894921,  0.00237881,  0.02861445, ...,  0.05236435,
        -0.03568534, -0.02537734],
       [-0.01517698, -0.1817214 ,  0.00510799, ..., -0.1363022 ,
         0.07744772, -0.04587692],
       [-0.0719701 , -0.03862236,  0.00605929, ..., -0.0009384 ,
        -0.01894928,  0.0116022 ],
       ...,
       [ 0.02291214,  0.03107316,  0.0581912 , ..., -0.00307031,
        -0.01773866,  0.05663324],
       [-0.00443126, -0.03759074, -0.00107925, ...,  0.0357363 ,
        -0.0214347 , -0.01909814],
       [ 0.02096119,  0.00543365, -0.00405475, ..., -0.04826686,
        -0.03115325, -0.00325373]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': Array([-0.00180957,  0.01529409,  0.01958043, ..., -0.00829968,
       -0.02563367,  0.00275606], dtype=float32),
                    'scale': Array([-0.0186807 ,  0.00872798,  0.02538491, ...,  0.0055809 ,
        0.02315959, -0.01200881], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': Array([1.1680623, 1.157661 , 1.0796328, ..., 1.1467127, 1.1279683,
       1.1683893], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': Array([[ 4.6610608e-04, -9.2302234e-04,  1.9256090e-04, ...,
        -1.8762015e-03,  3.1015230e-03,  7.7282086e-05],
       [ 4.2546907e-04,  1.2939374e-03,  4.0687714e-03, ...,
        -2.2190525e-03, -2.9033702e-03,  4.7080475e-04],
       [ 3.6429802e-03,  1.0424472e-03, -4.5762388e-03, ...,
         3.9652204e-03,  1.9283589e-03, -5.2442768e-04],
       ...,
       [-1.1196211e-03,  2.0621186e-03, -1.0337863e-03, ...,
         4.7227193e-04,  1.5723511e-03, -2.1845263e-03],
       [ 2.8016064e-03, -1.5755885e-03, -1.3314369e-03, ...,
         1.4067355e-03, -4.0574753e-04,  7.7799556e-04],
       [-3.2866246e-03,  4.4263457e-03, -3.2770564e-04, ...,
         1.1289970e-04, -3.6392491e-03,  6.3449412e-04]], dtype=float32),
                    'dora_m': Array([[[0.5322976 , 0.59634024, 0.5319944 , ..., 0.5444116 ,
         0.5711225 , 0.50693095],
        [0.7427808 , 0.51039016, 0.47084868, ..., 0.5236821 ,
         0.4503722 , 0.537869  ],
        [0.50433683, 0.47187275, 0.5480413 , ..., 0.5155916 ,
         0.5520279 , 0.6731289 ],
        ...,
        [0.584992  , 0.62776953, 0.4949407 , ..., 0.50828815,
         0.5093723 , 0.57163346],
        [0.51302105, 0.47567555, 0.58759695, ..., 0.602918  ,
         0.65152615, 0.5224054 ],
        [0.48854473, 0.48251405, 0.6112275 , ..., 0.58026534,
         0.5435507 , 0.6067749 ]]], dtype=float32),
                    'lora_a': Array([[-0.10847801],
       [ 0.29454434],
       [-0.26035413],
       ...,
       [-0.7982697 ],
       [ 0.73765343],
       [-0.08401869]], dtype=float32),
                    'lora_b': Array([[ 1.2094934e-05],
       [-6.4751603e-06],
       [ 6.3174293e-06],
       ...,
       [-2.2625470e-06],
       [-2.9553144e-07],
       [ 1.7485587e-05]], dtype=float32),
                    'w': Array([[[-1.08706683e-03,  4.60137837e-02,  8.90870579e-03, ...,
          1.76706687e-02,  9.42360982e-03,  2.26893439e-03],
        [-2.84814835e-02,  2.05505025e-02,  2.50078719e-02, ...,
         -8.25714320e-03, -1.54784461e-02,  1.57097764e-02],
        [ 2.61329673e-02, -1.43390916e-05, -1.57456137e-02, ...,
          5.81572438e-03,  9.63232853e-03,  3.29728536e-02],
        ...,
        [-2.58284789e-02, -6.61483314e-03,  4.17501107e-03, ...,
          1.56273600e-03, -1.34084104e-02, -2.00806791e-03],
        [ 2.15532375e-03,  5.60544012e-03, -1.21262446e-02, ...,
          1.41353793e-02,  1.00403710e-03, -2.26156227e-02],
        [ 6.50632137e-04,  3.77634016e-04,  9.30224918e-03, ...,
          2.15902999e-02, -2.99683325e-02,  4.09428496e-03]],

...,

[[ 7.57395895e-03, -8.69783759e-03, -6.51977910e-03, ...,
         -7.83248991e-03,  1.37400422e-02,  9.63847339e-03],
        [ 1.93790216e-02,  1.15005560e-02,  2.57335790e-03, ...,
         -1.17511407e-03, -1.32754035e-02,  2.84431316e-02],
        [ 8.90493859e-03, -6.35170203e-04,  1.79159716e-02, ...,
         -7.44165573e-03, -1.16720190e-02, -6.11115992e-03],
        ...,
        [-1.76805817e-02, -1.92710981e-02,  2.11877096e-02, ...,
         -1.47993183e-02, -2.90360246e-02, -2.44445037e-02],
        [ 2.60678609e-03,  1.54918600e-02,  7.54820416e-03, ...,
          1.60006043e-02, -3.96102592e-02, -1.15687342e-03],
        [ 1.01102330e-02, -1.23707214e-02, -7.94599764e-03, ...,
          4.61885557e-02, -6.15432067e-03,  2.03162176e-03]]],      dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': Array([0.3423228 , 0.37805688, 0.38570723, 0.3631645 , 0.3697752 ,
       0.38898537, 0.3462277 , 0.40135533, 0.32160336, 0.32755578,
       0.36695844, 0.38510692, 0.37241417, 0.36000612, 0.40425485,
       0.39355522, 0.37704387, 0.34560984, 0.39334768, 0.34970805,
       0.41583103, 0.32193702, 0.39897722, 0.38455772, 0.3881263 ,
       0.36545965, 0.30030486, 0.3719625 , 0.3724528 , 0.36381948,
       0.3207002 , 0.3917756 , 0.37410504, 0.35788256, 0.3723777 ,
       0.40420017, 0.30146837, 0.33636385, 0.37770233, 0.34213907,
       0.43725038, 0.35701525, 0.36562383, 0.32315183, 0.41536447,
       0.35835594, 0.34123057, 0.36961076, 0.32459491, 0.3978243 ,
       0.44356823, 0.41040212, 0.35807064, 0.37811923, 0.401245  ,
       0.38634533, 0.37138543, 0.33645675, 0.36295387, 0.35502782,
       0.36917937, 0.3897106 , 0.37134442, 0.39309275, 0.36695778,
       0.41020274, 0.44558018, 0.36960307, 0.29633558, 0.35949242,
       0.30078134, 0.37817764, 0.2984029 , 0.3855079 , 0.34376156,
       0.34066135, 0.34903902, 0.34249932, 0.35855636, 0.42332006],      dtype=float32)
                },
                'post': {
                    'b': Array([ 0.0003246 ,  0.00847099,  0.01464174, ..., -0.0033611 ,
       -0.00732129,  0.00176842], dtype=float32),
                    'dora_m': Array([[[0.4728817 , 0.42918554, 0.46712494, ..., 0.46857873,
         0.46419927, 0.47034577],
        [0.45037478, 0.44934806, 0.46160123, ..., 0.45701852,
         0.4529975 , 0.43453345],
        [0.46285325, 0.434797  , 0.4514307 , ..., 0.4655688 ,
         0.4507521 , 0.44786975],
        ...,
        [0.44068918, 0.44276518, 0.4716284 , ..., 0.4386408 ,
         0.45881268, 0.47704235],
        [0.39122215, 0.48147756, 0.4458559 , ..., 0.4753054 ,
         0.4541895 , 0.47319788],
        [0.407939  , 0.45421672, 0.44628716, ..., 0.4783134 ,
         0.47363475, 0.43661043]]], dtype=float32),
                    'lora_a': Array([[-0.10846515],
       [ 0.29453433],
       [-0.26038727],
       ...,
       [-0.7982783 ],
       [ 0.7376183 ],
       [-0.08400349]], dtype=float32),
                    'lora_b': Array([[ 8.8611923e-06],
       [ 4.8396682e-06],
       [ 6.2339709e-06],
       ...,
       [-5.0779913e-06],
       [-1.6287624e-05],
       [-1.0427792e-05]], dtype=float32),
                    'w': Array([[[-0.01333189, -0.00949195, -0.00053049, ...,  0.01240137,
         -0.00696176,  0.00108714],
        [-0.01398571, -0.00581668,  0.01127646, ...,  0.01062614,
          0.01003156, -0.0046383 ],
        [ 0.00261621, -0.00092907,  0.02294742, ..., -0.00945895,
          0.00738646,  0.01177557],
        ...,
        [ 0.03838847, -0.00842606, -0.00133155, ..., -0.00734077,
          0.01323657,  0.02828315],
        [-0.03245032,  0.00308438,  0.00897119, ...,  0.00759104,
         -0.00044007, -0.01147511],
        [ 0.00633265, -0.01913508,  0.00784844, ...,  0.00480823,
          0.00590358, -0.00663009]],

...,

[[ 0.00438632,  0.00552457,  0.01929554, ...,  0.00988519,
         -0.00359951, -0.00395988],
        [-0.01838505, -0.01033139,  0.00202073, ..., -0.00086231,
          0.00257837, -0.00091347],
        [-0.01157259,  0.00156012,  0.01622672, ...,  0.00595554,
         -0.00686237,  0.00272262],
        ...,
        [-0.02153967,  0.01909429, -0.02143722, ..., -0.02285551,
         -0.00333509, -0.00466016],
        [-0.00993951,  0.01258533,  0.00091125, ...,  0.01065476,
          0.01349149,  0.01044108],
        [ 0.01078804,  0.02377654, -0.00526888, ..., -0.01481672,
          0.0023681 , -0.02840961]]], dtype=float32)
                },
                'query': {
                    'b': Array([[ 1.68616569e-03,  7.35044386e-03, -7.82678742e-03, ...,
         1.13712074e-02, -1.20169930e-02, -7.39986310e-03],
       [-4.70847776e-03,  6.84696948e-03,  1.45763187e-02, ...,
        -1.89887341e-02, -1.99760124e-02, -8.20217654e-03],
       [ 1.99295748e-02,  1.89537518e-02, -2.01193895e-02, ...,
         1.57649498e-02, -1.20130610e-02,  1.74873788e-02],
       ...,
       [-9.92237838e-05,  1.43216911e-03, -1.05841765e-02, ...,
         5.57100540e-03,  1.08484523e-02,  1.23333735e-02],
       [ 1.54685080e-02, -1.70757119e-02, -1.84352957e-02, ...,
         9.89105832e-03,  7.58174481e-03,  2.33456977e-02],
       [-8.85467511e-03,  6.02648500e-03, -4.42392053e-03, ...,
        -1.57583761e-03, -5.54478436e-04, -8.62513727e-04]],      dtype=float32),
                    'dora_m': Array([[[0.7820993 , 0.83724236, 0.78103346, ..., 0.9300915 ,
         0.9935601 , 0.87842005],
        [0.6553627 , 0.8105458 , 0.91440505, ..., 0.7932846 ,
         0.8616683 , 0.71699446],
        [0.88193285, 0.7470871 , 0.8440272 , ..., 0.72470564,
         0.83062834, 0.7472537 ],
        ...,
        [0.745487  , 0.78920615, 0.76723456, ..., 0.78710246,
         0.7412791 , 0.789311  ],
        [0.8562675 , 0.63015646, 0.6638248 , ..., 0.7610497 ,
         0.7265305 , 0.69988626],
        [0.8348919 , 0.72874737, 0.80342644, ..., 0.83797216,
         0.83486056, 0.6417563 ]]], dtype=float32),
                    'lora_a': Array([[-0.1084516 ],
       [ 0.29455253],
       [-0.26035136],
       ...,
       [-0.7982782 ],
       [ 0.7376268 ],
       [-0.08403474]], dtype=float32),
                    'lora_b': Array([[-1.7736689e-05],
       [ 1.7919965e-05],
       [ 1.3622246e-05],
       ...,
       [ 2.0807794e-05],
       [ 1.2836763e-05],
       [ 1.9377998e-05]], dtype=float32),
                    'w': Array([[[ 0.03277196, -0.02859401, -0.00419887, ...,  0.03090124,
          0.05109725,  0.01152376],
        [ 0.00334124,  0.02527811,  0.02353178, ..., -0.00263442,
          0.00997576,  0.01351473],
        [-0.0116917 , -0.01465859, -0.00956394, ...,  0.01710192,
         -0.00084864,  0.01143387],
        ...,
        [-0.01677076, -0.00378386, -0.0149406 , ...,  0.00874252,
          0.02506459,  0.00360843],
        [ 0.01314708,  0.0301985 ,  0.00475084, ...,  0.02091082,
          0.03415568,  0.0144391 ],
        [-0.04189856,  0.02725412,  0.01354138, ...,  0.01726145,
         -0.06039976,  0.01708226]],

...,

[[-0.02100616, -0.01229226, -0.00664236, ..., -0.00397686,
         -0.01140649,  0.01656644],
        [ 0.02318282,  0.00089357,  0.00351443, ..., -0.04290025,
         -0.02527255,  0.02310306],
        [ 0.00123981,  0.00110993,  0.02510653, ..., -0.05794868,
          0.02559463,  0.03155119],
        ...,
        [ 0.0041406 ,  0.03817632,  0.0238045 , ..., -0.01445156,
         -0.02611304,  0.0221843 ],
        [-0.03239568, -0.03269725,  0.04132884, ...,  0.01554982,
          0.02150906, -0.0006056 ],
        [-0.00081417, -0.0145265 ,  0.0345278 , ...,  0.01734272,
         -0.02692604, -0.0183093 ]]], dtype=float32)
                },
                'value': {
                    'b': Array([[-6.0506293e-04,  5.6609279e-03,  1.6824390e-04, ...,
         3.6446922e-03, -4.2956732e-03,  2.7651975e-03],
       [-1.9702064e-03,  5.6249095e-04,  1.8836118e-04, ...,
        -1.1110047e-03, -7.1162550e-04, -1.0728554e-03],
       [ 1.7920202e-03, -7.7035325e-04, -1.0077038e-04, ...,
        -5.4486137e-04,  1.0511113e-03, -1.3385109e-03],
       ...,
       [ 1.4025952e-03, -5.3501499e-05,  3.7820532e-03, ...,
         7.0839765e-04,  1.5973275e-04, -7.4703916e-04],
       [ 2.3972094e-03,  1.8491159e-03, -2.3841648e-03, ...,
        -9.0237451e-04,  2.9343597e-03, -1.7331726e-03],
       [-2.2909330e-04,  3.7682767e-04,  6.2282931e-04, ...,
         7.3327468e-04, -1.5061294e-03,  6.9486239e-04]], dtype=float32),
                    'dora_m': Array([[[0.51123023, 0.44956207, 0.46779463, ..., 0.48365465,
         0.47331136, 0.49376735],
        [0.4677435 , 0.46323866, 0.46745342, ..., 0.4641058 ,
         0.47963774, 0.4603188 ],
        [0.49710435, 0.45096913, 0.4894225 , ..., 0.47861296,
         0.4836678 , 0.47351474],
        ...,
        [0.47489664, 0.45720083, 0.49065268, ..., 0.47888604,
         0.4950038 , 0.47051618],
        [0.3958552 , 0.5141372 , 0.46221778, ..., 0.46745428,
         0.506505  , 0.5051713 ],
        [0.43391252, 0.47447917, 0.47726923, ..., 0.51171243,
         0.48292935, 0.4566678 ]]], dtype=float32),
                    'lora_a': Array([[-0.10844978],
       [ 0.294552  ],
       [-0.26035008],
       ...,
       [-0.7982786 ],
       [ 0.73761666],
       [-0.08403903]], dtype=float32),
                    'lora_b': Array([[-1.6946335e-05],
       [-1.2465810e-05],
       [ 1.5380647e-05],
       ...,
       [ 9.0137346e-06],
       [ 1.3055999e-05],
       [-6.9661196e-06]], dtype=float32),
                    'w': Array([[[ 1.45941954e-02,  3.29494383e-03,  3.19986517e-04, ...,
         -9.69769619e-03,  1.06396377e-02, -1.90019403e-02],
        [ 1.43488497e-02, -9.36437212e-03,  4.04258724e-03, ...,
         -6.97107939e-03,  6.69743586e-03,  4.20573447e-03],
        [-2.41478905e-03,  1.75840594e-02, -1.30269798e-02, ...,
         -5.77327935e-03,  1.25292856e-02, -2.13064831e-02],
        ...,
        [-2.55727731e-02, -5.48362406e-03, -4.82275616e-03, ...,
         -1.88632980e-02, -7.25970929e-03,  1.63748935e-02],
        [-4.61501162e-03, -1.54769775e-02,  9.79034416e-03, ...,
         -3.15764919e-03,  1.46130370e-02,  1.14133917e-02],
        [ 2.69468836e-02, -1.04775866e-02, -8.37587751e-03, ...,
          1.32150073e-02, -1.61835197e-02,  1.91893261e-02]],

...,

[[ 1.83889840e-03, -9.06582084e-03,  8.98056477e-03, ...,
         -3.42143402e-02,  1.34969996e-02, -2.68482720e-03],
        [ 1.95417255e-02,  8.69073067e-03, -1.31595889e-02, ...,
         -3.40937357e-03,  1.11017330e-03, -2.54486352e-02],
        [ 4.01821807e-02, -1.65785868e-02, -1.30676490e-03, ...,
          9.75302607e-03,  8.82033538e-03, -1.49140591e-02],
        ...,
        [-5.97554166e-03,  2.31686104e-02,  1.02580320e-02, ...,
          6.09135488e-03,  1.25123495e-02, -2.34593637e-02],
        [ 1.90906879e-02, -3.57933566e-02, -4.43168171e-02, ...,
         -2.67305952e-02,  5.62132103e-03, -9.73594468e-03],
        [-5.58904791e-03,  1.43911988e-02, -5.09532820e-03, ...,
          2.70975586e-02,  2.44144676e-03,  9.71086882e-03]]],      dtype=float32)
                }
            }
        },
        'x_layers_14': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': Array([ 0.00029601, -0.01084311, -0.00675669, ..., -0.01585302,
       -0.01105882, -0.00785637], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[0.9577501, 1.1869366, 1.117573 , ..., 1.1598082, 1.0978234,
        1.0929182]], dtype=float32),
                        'lora_a': Array([[-0.10844997],
       [ 0.29455447],
       [-0.26035124],
       ...,
       [-0.79826593],
       [ 0.7376555 ],
       [-0.08399895]], dtype=float32),
                        'lora_b': Array([[ 1.7449045e-05],
       [-1.2247361e-05],
       [-1.6881335e-05],
       ...,
       [-1.4348217e-05],
       [-1.8831040e-06],
       [ 1.4693893e-05]], dtype=float32),
                        'w': Array([[ 0.00582105,  0.01318311,  0.03708626, ..., -0.01971904,
        -0.01395077,  0.02952531],
       [ 0.00619035, -0.03169769,  0.01273892, ...,  0.01644244,
         0.01044564, -0.03745968],
       [-0.03273934,  0.0426847 ,  0.01360384, ..., -0.03426943,
        -0.03494504, -0.04589375],
       ...,
       [-0.02636916,  0.06461645,  0.00987081, ...,  0.00360583,
         0.01588036, -0.01292699],
       [ 0.02911054,  0.08070075,  0.03281379, ...,  0.02382054,
        -0.00493196,  0.01056881],
       [ 0.03009207, -0.04479855,  0.00666338, ...,  0.04239031,
         0.02533692,  0.023514  ]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': Array([ 0.00158921,  0.0018311 ,  0.00338582, ..., -0.00117   ,
       -0.00201284, -0.00039685], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.2462132, 1.3154612, 1.1777734, ..., 1.2787412, 1.2724178,
        1.3852592]], dtype=float32),
                        'lora_a': Array([[-0.10849126],
       [ 0.29453158],
       [-0.26038247],
       ...,
       [-0.7982616 ],
       [ 0.7376338 ],
       [-0.0840209 ]], dtype=float32),
                        'lora_b': Array([[ 1.8641190e-06],
       [-1.5275962e-05],
       [-1.7270639e-05],
       ...,
       [-1.3897887e-05],
       [-9.9244235e-06],
       [ 1.6422155e-05]], dtype=float32),
                        'w': Array([[ 0.01624611,  0.0337978 ,  0.02114537, ..., -0.00867737,
        -0.02622462, -0.00260242],
       [ 0.01332312,  0.11018921, -0.04020173, ...,  0.00130645,
         0.04434073,  0.00471618],
       [ 0.03531237,  0.02962698, -0.02800041, ...,  0.00812992,
         0.05103358,  0.02820559],
       ...,
       [-0.05357926, -0.04292846,  0.01705871, ...,  0.02818025,
        -0.0585423 , -0.01409262],
       [ 0.02043611, -0.06592052, -0.02197525, ...,  0.01451133,
        -0.03996396, -0.02293865],
       [-0.02165488,  0.05503482,  0.06325393, ..., -0.02189652,
         0.01855572,  0.01316425]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': Array([ 0.00385938,  0.01357866,  0.02887791, ..., -0.00625249,
       -0.02258822,  0.01229282], dtype=float32),
                    'scale': Array([-0.01239096,  0.0149999 ,  0.01164342, ...,  0.01213837,
        0.03059725,  0.00806229], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': Array([1.0972409, 1.1051496, 1.1049464, ..., 1.121043 , 1.1114082,
       1.118717 ], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': Array([[ 1.25841633e-03,  1.55034452e-03,  1.08872564e-03, ...,
        -9.54594580e-04, -1.65263680e-03, -4.30746097e-03],
       [-1.98853505e-03,  6.85305131e-05, -1.63052906e-03, ...,
         4.81848343e-04,  2.22792180e-04, -1.81035930e-03],
       [ 5.71344688e-04,  1.74748956e-03,  1.30943884e-03, ...,
        -4.07261134e-04, -1.63442385e-03,  8.52171797e-04],
       ...,
       [ 1.20874020e-05,  1.84527622e-03,  8.76582169e-04, ...,
        -6.50884467e-04,  2.91718403e-03, -2.68512813e-04],
       [ 1.46702142e-03,  1.36173528e-03, -1.60505017e-03, ...,
         1.72068970e-03, -3.37592070e-03,  1.48777233e-03],
       [-2.02768855e-03,  1.06071595e-04,  9.18209844e-04, ...,
        -1.73310947e-03, -8.43419926e-04, -2.83935195e-04]],      dtype=float32),
                    'dora_m': Array([[[0.64241683, 0.5794385 , 0.6329514 , ..., 0.56769484,
         0.5796082 , 0.4911193 ],
        [0.5286798 , 0.57554615, 0.47254   , ..., 0.6169602 ,
         0.5901556 , 0.6126638 ],
        [0.5938834 , 0.54917246, 0.6326621 , ..., 0.5949766 ,
         0.531918  , 0.6551565 ],
        ...,
        [0.570728  , 0.7721166 , 0.6250424 , ..., 0.5541942 ,
         0.5575996 , 0.5603657 ],
        [0.58110034, 0.53208053, 0.48675606, ..., 0.47185954,
         0.46097937, 0.49673802],
        [0.549224  , 0.7201666 , 0.5543733 , ..., 0.5999769 ,
         0.6006742 , 0.5997057 ]]], dtype=float32),
                    'lora_a': Array([[-0.10847919],
       [ 0.29455096],
       [-0.26038387],
       ...,
       [-0.79827446],
       [ 0.7376508 ],
       [-0.0840008 ]], dtype=float32),
                    'lora_b': Array([[-1.20948525e-05],
       [-1.14987661e-05],
       [-1.14496988e-05],
       ...,
       [-2.01237490e-05],
       [-1.09243583e-05],
       [-5.99692476e-06]], dtype=float32),
                    'w': Array([[[ 0.01589175,  0.03752959,  0.00500297, ..., -0.02840356,
         -0.01997883, -0.02230303],
        [ 0.02896536, -0.00461972, -0.00667112, ..., -0.02033726,
          0.01411753,  0.01882261],
        [-0.00266081,  0.00936987,  0.0201344 , ...,  0.00801549,
          0.00261194,  0.01681161],
        ...,
        [ 0.00451914, -0.0110497 ,  0.00833845, ...,  0.00385557,
         -0.00774913,  0.02446727],
        [ 0.03294024,  0.01529131, -0.02611599, ...,  0.01307458,
         -0.01864656,  0.00500216],
        [ 0.00635432,  0.03710559,  0.0123241 , ...,  0.01083144,
          0.0166588 , -0.00347141]],

...,

[[ 0.00333968,  0.02958334, -0.00317894, ...,  0.00457225,
         -0.0026831 , -0.00992109],
        [-0.02278157,  0.0058978 ,  0.02510419, ..., -0.01814865,
          0.00928549,  0.02088999],
        [-0.01182269, -0.00030995, -0.0023372 , ...,  0.00272881,
         -0.00585036,  0.02321105],
        ...,
        [ 0.00483019,  0.02018814,  0.02192444, ..., -0.02810882,
         -0.00201883, -0.00477643],
        [-0.0016086 ,  0.00650351, -0.02945073, ...,  0.01685349,
         -0.01316662,  0.00660059],
        [ 0.03185076, -0.02384418,  0.00400093, ...,  0.00800042,
          0.00193645, -0.01411922]]], dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': Array([0.37061766, 0.38822898, 0.35322222, 0.34075966, 0.2893336 ,
       0.33207327, 0.36952117, 0.31159094, 0.40695235, 0.3650621 ,
       0.30505344, 0.32530743, 0.38286367, 0.30215907, 0.37921858,
       0.31768548, 0.35374323, 0.27960575, 0.33220676, 0.27091444,
       0.33243063, 0.35063368, 0.33900893, 0.3868279 , 0.35574022,
       0.3405869 , 0.3743824 , 0.30515498, 0.38332865, 0.3609494 ,
       0.3061473 , 0.33329   , 0.3651566 , 0.3100548 , 0.3234738 ,
       0.3393868 , 0.37142864, 0.35722014, 0.30790803, 0.3584052 ,
       0.3168645 , 0.30959433, 0.25981882, 0.31101865, 0.32634166,
       0.38462982, 0.31919768, 0.29806888, 0.35299852, 0.37269595,
       0.35165468, 0.41325662, 0.3530622 , 0.36405906, 0.31029075,
       0.38071197, 0.38286918, 0.33056083, 0.35136825, 0.26401302,
       0.36774495, 0.32272413, 0.3591407 , 0.26142058, 0.26791415,
       0.3675712 , 0.39433083, 0.4035874 , 0.38473964, 0.37995642,
       0.31737867, 0.4083564 , 0.37195632, 0.36349788, 0.34845665,
       0.32651016, 0.36170384, 0.29348466, 0.35695028, 0.35530826],      dtype=float32)
                },
                'post': {
                    'b': Array([ 0.00137819,  0.00906399,  0.01699077, ..., -0.00279563,
       -0.00908101,  0.00332175], dtype=float32),
                    'dora_m': Array([[[0.45040956, 0.4695895 , 0.4423614 , ..., 0.4413326 ,
         0.47642618, 0.48618788],
        [0.47195008, 0.48969197, 0.49020362, ..., 0.48478597,
         0.50448895, 0.47670987],
        [0.43910205, 0.46792784, 0.38730007, ..., 0.4821558 ,
         0.4633401 , 0.47592995],
        ...,
        [0.45133084, 0.46696806, 0.41731533, ..., 0.46461412,
         0.4669114 , 0.47295007],
        [0.4362905 , 0.4408165 , 0.42923763, ..., 0.44580263,
         0.44040403, 0.4418033 ],
        [0.44837123, 0.4609461 , 0.47698835, ..., 0.4364064 ,
         0.41478777, 0.4389704 ]]], dtype=float32),
                    'lora_a': Array([[-0.10848371],
       [ 0.29451606],
       [-0.26035768],
       ...,
       [-0.7982784 ],
       [ 0.73761904],
       [-0.08400354]], dtype=float32),
                    'lora_b': Array([[-8.1693888e-06],
       [ 9.3393537e-06],
       [ 1.6660420e-05],
       ...,
       [-1.6948605e-05],
       [ 8.0774007e-06],
       [ 2.0745103e-05]], dtype=float32),
                    'w': Array([[[ 0.0077813 , -0.0187567 ,  0.00135137, ..., -0.0074239 ,
         -0.00509173,  0.0028086 ],
        [-0.01502111,  0.01735665, -0.01783477, ..., -0.01821919,
          0.02553056, -0.0240744 ],
        [ 0.02150158, -0.00834437, -0.01518894, ...,  0.00089701,
         -0.00358897, -0.01120061],
        ...,
        [-0.01904649,  0.01619412, -0.03076166, ...,  0.00734015,
         -0.02245972, -0.00764283],
        [-0.00526892,  0.00547887, -0.0033242 , ...,  0.00683106,
          0.00080588,  0.01187587],
        [-0.00852314, -0.01253605, -0.01547347, ..., -0.004784  ,
         -0.01609213, -0.00491303]],

...,

[[ 0.01126035,  0.01273246, -0.01011568, ..., -0.01767948,
         -0.00159439, -0.0073643 ],
        [ 0.0052752 , -0.00572318, -0.01242626, ...,  0.00613892,
         -0.03157397,  0.00018649],
        [-0.00516451, -0.00544372,  0.00276264, ...,  0.00999606,
         -0.01028886, -0.00110074],
        ...,
        [ 0.01464988, -0.00893058, -0.00986978, ..., -0.00508787,
         -0.02117784, -0.0030193 ],
        [-0.00746697, -0.01532643, -0.00941347, ..., -0.00074038,
         -0.00212986,  0.02092797],
        [-0.01018735,  0.01143691,  0.01311445, ..., -0.01059256,
         -0.01391992,  0.00311824]]], dtype=float32)
                },
                'query': {
                    'b': Array([[ 0.005839  ,  0.00983257,  0.0143399 , ..., -0.01401017,
        -0.02116291, -0.02012843],
       [ 0.01007332,  0.00880442,  0.01877066, ...,  0.00415033,
        -0.01021724,  0.00293849],
       [-0.00301371, -0.00309211, -0.00566929, ...,  0.00209435,
        -0.00242971,  0.00093665],
       ...,
       [ 0.00573737,  0.00622756,  0.00125796, ..., -0.01130205,
         0.01154514,  0.00515189],
       [ 0.01658523,  0.00806783, -0.01109188, ...,  0.01466294,
        -0.0154193 ,  0.01680345],
       [ 0.00661552, -0.00166651, -0.00556861, ...,  0.00759342,
         0.00582669, -0.00205902]], dtype=float32),
                    'dora_m': Array([[[0.8983354 , 0.96500987, 0.8915905 , ..., 0.93076706,
         0.9834379 , 0.8508025 ],
        [0.93370545, 0.70447564, 0.8168961 , ..., 0.7803381 ,
         0.8181559 , 0.80786747],
        [0.9835088 , 1.0560386 , 0.93962634, ..., 0.9270827 ,
         1.1537958 , 0.8445198 ],
        ...,
        [0.7043295 , 0.6004237 , 0.82326615, ..., 0.933709  ,
         0.8300482 , 0.75901556],
        [0.74353576, 0.7008146 , 0.72558486, ..., 0.77498406,
         0.60737365, 0.9083807 ],
        [0.81321853, 0.6770651 , 0.8139339 , ..., 0.7610465 ,
         0.8407953 , 0.8276135 ]]], dtype=float32),
                    'lora_a': Array([[-0.10844937],
       [ 0.29455253],
       [-0.26035416],
       ...,
       [-0.7982778 ],
       [ 0.7376196 ],
       [-0.08400024]], dtype=float32),
                    'lora_b': Array([[-5.9482227e-07],
       [-2.9002711e-06],
       [ 4.2889733e-06],
       ...,
       [-1.1481540e-05],
       [-1.9865822e-05],
       [-2.0603222e-05]], dtype=float32),
                    'w': Array([[[ 4.31832578e-03,  4.43392396e-02, -1.94164272e-02, ...,
         -1.54080531e-02, -2.04299670e-02, -5.17385080e-03],
        [ 1.04202796e-02, -3.61938798e-03,  2.09692549e-02, ...,
          9.26325377e-03, -1.67021919e-02,  1.68145522e-02],
        [ 1.27045680e-02,  2.11668964e-02, -1.38618518e-02, ...,
         -2.85082720e-02,  4.86320928e-02, -2.15413631e-03],
        ...,
        [-9.61130206e-03, -1.08617987e-03,  4.71109618e-03, ...,
          1.04664522e-03,  6.03351854e-02, -5.40138781e-03],
        [ 2.92769670e-02,  3.52728590e-02, -3.96639816e-02, ...,
         -1.35405455e-02, -8.60308576e-03,  2.50309445e-02],
        [-2.57780645e-02, -2.47719996e-02,  7.15856850e-02, ...,
         -1.76313594e-02, -3.09293456e-02,  1.78850945e-02]],

...,

[[ 1.74412504e-02,  2.77380887e-02,  2.66704592e-03, ...,
          1.13621624e-02,  3.15764658e-02,  1.05727613e-02],
        [-2.83396821e-02, -2.88270274e-03,  2.99232360e-02, ...,
          6.64325291e-03,  2.54750084e-02,  1.79602876e-02],
        [-4.25079698e-03,  1.65656265e-02, -5.48844039e-02, ...,
          4.05983292e-02,  2.32505482e-02,  2.22766977e-02],
        ...,
        [-3.46966344e-03, -4.04150458e-03, -5.48522398e-02, ...,
         -1.38870236e-02,  2.99531091e-02,  5.51477168e-03],
        [-2.55604777e-02,  1.75041482e-02,  2.97506843e-02, ...,
         -2.03855336e-02, -9.80742552e-05,  3.82814668e-02],
        [ 5.42269973e-03, -8.90803058e-03, -1.50118424e-02, ...,
         -2.57738512e-02, -2.58772653e-02,  8.77469871e-03]]],      dtype=float32)
                },
                'value': {
                    'b': Array([[-3.8052234e-03,  3.6348451e-03, -2.1750149e-03, ...,
         3.6451784e-03,  6.7220966e-04, -2.0147774e-03],
       [-2.4101972e-03, -1.9176434e-03,  2.1671599e-03, ...,
        -1.3661172e-03, -1.3818906e-03, -8.1987807e-04],
       [-1.6283204e-03, -4.0112906e-03,  2.5617480e-03, ...,
        -3.6272124e-04, -1.1886849e-04, -2.5138080e-03],
       ...,
       [-2.4189497e-03, -5.2452169e-04,  4.6270331e-03, ...,
        -7.0718667e-03, -3.3432364e-03, -1.8380929e-03],
       [ 1.5326069e-03, -1.4740487e-03,  3.8775629e-03, ...,
        -9.5384319e-05, -6.2146760e-04, -2.4011079e-03],
       [ 1.3725453e-03,  1.5336208e-03, -7.2143367e-04, ...,
         9.6710742e-04,  4.9143750e-03, -2.6023157e-03]], dtype=float32),
                    'dora_m': Array([[[0.45901498, 0.510779  , 0.46130732, ..., 0.44356182,
         0.49134892, 0.5227705 ],
        [0.49013746, 0.5012387 , 0.4971409 , ..., 0.47284994,
         0.5097601 , 0.49357396],
        [0.44283056, 0.47481754, 0.39203447, ..., 0.45666185,
         0.4586499 , 0.4854515 ],
        ...,
        [0.49135482, 0.48741925, 0.43081614, ..., 0.48801592,
         0.5086796 , 0.49352196],
        [0.4589756 , 0.4687593 , 0.47355193, ..., 0.48188004,
         0.4652063 , 0.49884403],
        [0.46564242, 0.47816315, 0.49098068, ..., 0.47932804,
         0.43669224, 0.46232286]]], dtype=float32),
                    'lora_a': Array([[-0.10845097],
       [ 0.2945535 ],
       [-0.26035115],
       ...,
       [-0.79827726],
       [ 0.73761386],
       [-0.08399942]], dtype=float32),
                    'lora_b': Array([[-1.3998178e-05],
       [-1.7014268e-05],
       [ 1.0470997e-05],
       ...,
       [ 2.1420279e-05],
       [ 1.0085087e-05],
       [-1.1184281e-05]], dtype=float32),
                    'w': Array([[[-1.70202069e-02, -1.97600815e-02, -1.55001692e-02, ...,
         -8.71328916e-03,  1.83615531e-03,  9.72294621e-03],
        [ 1.58263091e-02,  6.49763329e-04,  1.71479937e-02, ...,
         -5.63265057e-03, -2.26127431e-02,  3.16370688e-02],
        [-3.67097347e-03, -1.10905785e-02,  1.21303340e-02, ...,
          1.62765887e-02,  2.08019326e-03,  8.82455520e-03],
        ...,
        [-2.60162316e-02,  2.52318997e-02,  2.38402616e-02, ...,
          1.30455270e-02, -1.14394212e-02,  3.46929883e-05],
        [ 9.73733980e-03,  4.70478972e-03,  1.08287651e-02, ...,
         -7.70605751e-04,  1.00928349e-02,  1.45578319e-02],
        [-9.30155907e-03,  1.85461044e-02, -3.05988593e-03, ...,
         -4.52069426e-03, -1.08060869e-03, -1.10367388e-02]],

...,

[[-4.07601148e-03,  1.09111238e-02,  1.46400845e-02, ...,
          1.44178495e-02, -1.41544109e-02, -1.25444625e-02],
        [-5.62398247e-02, -7.10166758e-04,  6.04600087e-03, ...,
          3.83043638e-03,  1.28686754e-02,  1.30163999e-02],
        [-3.78315017e-04, -8.86098854e-03, -9.61992983e-03, ...,
          2.75602285e-03,  5.36391279e-03,  5.96022047e-03],
        ...,
        [ 6.56709680e-03,  1.64176356e-02, -4.17928863e-03, ...,
         -4.09208611e-03, -4.79598151e-04, -7.82177225e-03],
        [ 2.35127136e-02, -2.28902791e-03, -5.07572992e-03, ...,
          4.67127515e-03, -7.65140029e-03, -4.28619620e-04],
        [ 2.34100297e-02,  2.30132253e-03,  3.33477068e-03, ...,
          4.12812969e-03,  9.00776312e-03,  2.72210352e-02]]],      dtype=float32)
                }
            }
        },
        'x_layers_15': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': Array([-0.00382897, -0.01070694, -0.00421839, ..., -0.00438798,
       -0.00500148, -0.0174924 ], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.0075961, 1.0798019, 1.0374566, ..., 1.0977092, 1.0510852,
        1.2827349]], dtype=float32),
                        'lora_a': Array([[-0.10844995],
       [ 0.29455394],
       [-0.260355  ],
       ...,
       [-0.7982713 ],
       [ 0.73762995],
       [-0.0839987 ]], dtype=float32),
                        'lora_b': Array([[ 1.7099634e-05],
       [ 5.7212796e-06],
       [-1.7446750e-05],
       ...,
       [-2.0855876e-05],
       [-1.6475969e-05],
       [ 8.1851958e-06]], dtype=float32),
                        'w': Array([[-0.03001062, -0.04300353,  0.03624095, ...,  0.04447658,
         0.00738414, -0.00944565],
       [ 0.04650056,  0.02834052, -0.03979881, ..., -0.00881591,
        -0.05368604, -0.01600092],
       [-0.04603452,  0.00333315,  0.02914217, ...,  0.02335356,
         0.02082835, -0.01562827],
       ...,
       [ 0.0200206 , -0.02080444, -0.05211437, ...,  0.07481808,
         0.03825745,  0.04028043],
       [ 0.00603536, -0.00296809,  0.00588007, ...,  0.01323494,
        -0.009428  ,  0.01075984],
       [-0.02157067,  0.00690726,  0.00761885, ..., -0.03510019,
         0.02382934, -0.0114434 ]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': Array([-0.0034491 ,  0.00113755,  0.00326148, ..., -0.00202693,
       -0.00075667, -0.00020701], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.3310684, 1.3019623, 1.1762334, ..., 1.3272754, 1.3381732,
        1.3899728]], dtype=float32),
                        'lora_a': Array([[-0.10846358],
       [ 0.2945345 ],
       [-0.2603512 ],
       ...,
       [-0.7982444 ],
       [ 0.7376441 ],
       [-0.08402307]], dtype=float32),
                        'lora_b': Array([[ 1.8091501e-05],
       [-1.5991745e-05],
       [-1.6715754e-05],
       ...,
       [ 2.9195555e-06],
       [-1.4512517e-05],
       [ 1.0216359e-05]], dtype=float32),
                        'w': Array([[ 0.02785643,  0.00816482,  0.00832493, ..., -0.01484761,
        -0.01775872, -0.0023947 ],
       [-0.02799329, -0.0354571 , -0.00193841, ..., -0.04426453,
        -0.04666989,  0.0139689 ],
       [-0.03626651,  0.02228458,  0.04096157, ...,  0.02852779,
         0.02718053, -0.02925706],
       ...,
       [-0.02451389, -0.02001832,  0.01434541, ..., -0.02924969,
        -0.02220753,  0.02742435],
       [-0.01701572, -0.0317556 , -0.01858513, ..., -0.05272435,
         0.02035795, -0.03656702],
       [-0.15323555,  0.01148045,  0.00145428, ..., -0.03268117,
         0.06788057, -0.00867513]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': Array([ 0.00694113,  0.0082615 ,  0.01453089, ..., -0.00314942,
       -0.02481697, -0.00868503], dtype=float32),
                    'scale': Array([ 0.00132329,  0.01813487,  0.01469828, ..., -0.01991167,
        0.01208597,  0.00790367], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': Array([1.1506488, 1.1302264, 1.1266121, ..., 1.1562704, 1.0905734,
       1.1557795], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': Array([[-0.00140497,  0.00156974, -0.00102715, ...,  0.00140835,
        -0.00496419, -0.00182753],
       [-0.00232494,  0.00249131, -0.0014481 , ...,  0.00208211,
        -0.00203303, -0.00034535],
       [ 0.000678  , -0.00365663,  0.00025134, ...,  0.00110168,
         0.00079804,  0.00055274],
       ...,
       [-0.00070817, -0.00151383, -0.0004914 , ..., -0.0033437 ,
        -0.00261433,  0.00016033],
       [-0.00116644, -0.00049791, -0.00098731, ..., -0.00085122,
        -0.00029645,  0.00071196],
       [-0.00285295,  0.0006996 ,  0.00339124, ..., -0.00067429,
        -0.00202245, -0.00057098]], dtype=float32),
                    'dora_m': Array([[[0.58970374, 0.6016823 , 0.5226088 , ..., 0.69042265,
         0.4926946 , 0.6003331 ],
        [0.63332444, 0.5863573 , 0.59318686, ..., 0.5868602 ,
         0.57266355, 0.68414265],
        [0.645244  , 0.537967  , 0.60265434, ..., 0.5636669 ,
         0.58632225, 0.5243018 ],
        ...,
        [0.5388665 , 0.5505886 , 0.5699982 , ..., 0.5576157 ,
         0.5425719 , 0.5300792 ],
        [0.621468  , 0.6990387 , 0.78582907, ..., 0.54501206,
         0.83329284, 0.58033806],
        [0.5518827 , 0.5193674 , 0.53868467, ..., 0.53637993,
         0.5679438 , 0.61490303]]], dtype=float32),
                    'lora_a': Array([[-0.10848434],
       [ 0.29451287],
       [-0.26039046],
       ...,
       [-0.79827356],
       [ 0.7376383 ],
       [-0.0840076 ]], dtype=float32),
                    'lora_b': Array([[ 9.21172523e-06],
       [-6.63615128e-06],
       [ 1.74566976e-05],
       ...,
       [ 1.27962849e-05],
       [ 6.90763272e-06],
       [-1.05078525e-05]], dtype=float32),
                    'w': Array([[[-1.68806985e-02,  2.91784927e-02,  1.09832585e-02, ...,
          1.80186387e-02, -1.02492515e-02,  2.72594742e-03],
        [-1.41559094e-02, -3.16049792e-02,  3.01855132e-02, ...,
         -1.00077437e-02,  1.53082535e-02, -8.09445791e-03],
        [-2.88749132e-02,  8.75951163e-03,  2.48974888e-03, ...,
         -2.28136107e-02, -3.04544698e-02,  6.84893923e-03],
        ...,
        [ 1.17152147e-02, -9.77470540e-03,  1.43085346e-02, ...,
          1.60561968e-02,  2.40543135e-03,  2.78628282e-02],
        [-1.47178555e-02, -6.04795059e-04,  2.65847845e-03, ...,
         -2.50171544e-03, -3.12985815e-02, -1.53684812e-02],
        [-1.92413330e-02,  3.39501584e-03, -1.31989354e-02, ...,
         -1.14938617e-02, -7.27869244e-03, -3.29785720e-02]],

...,

[[-1.22254500e-02,  4.47095558e-03,  1.97416022e-02, ...,
         -1.80803724e-02,  2.45548575e-03,  1.53298164e-03],
        [-2.68470813e-02, -2.59301029e-02,  1.07112490e-02, ...,
          1.96190402e-02, -3.02873785e-04, -4.77690902e-03],
        [ 6.79769693e-03,  1.80204399e-02,  1.43247014e-02, ...,
          1.25488741e-02,  8.55445396e-03, -5.46903675e-03],
        ...,
        [-1.92873869e-02, -6.78052800e-03,  1.66579057e-02, ...,
          2.67223623e-02, -4.65476373e-03,  3.95291904e-03],
        [ 1.50315964e-03, -4.17142082e-03, -3.82395945e-02, ...,
         -1.29380785e-02, -1.02337543e-02, -1.88998599e-03],
        [-3.15461121e-02,  9.29084327e-03, -1.73337702e-02, ...,
         -1.78088769e-02, -9.98180453e-03, -2.37116124e-03]]],      dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': Array([0.3838729 , 0.32792246, 0.36369315, 0.3325522 , 0.33127862,
       0.37433836, 0.38163787, 0.36453605, 0.32643825, 0.36529392,
       0.32667154, 0.28192905, 0.3830394 , 0.47810373, 0.3558601 ,
       0.4301477 , 0.35760036, 0.3239827 , 0.34011665, 0.35472804,
       0.36542976, 0.3011298 , 0.28342828, 0.36540616, 0.39521998,
       0.33422124, 0.3204194 , 0.40146363, 0.3714171 , 0.3723778 ,
       0.35148665, 0.35613933, 0.36487457, 0.41195023, 0.3453005 ,
       0.3637106 , 0.37228388, 0.26457918, 0.38216782, 0.30997133,
       0.4377603 , 0.4159947 , 0.40109572, 0.3988191 , 0.39574093,
       0.23585324, 0.4167122 , 0.33482814, 0.3374693 , 0.3074873 ,
       0.36557922, 0.46011403, 0.29139513, 0.3209499 , 0.3571471 ,
       0.38119125, 0.4059438 , 0.4321849 , 0.37577084, 0.2836916 ,
       0.33823603, 0.33339486, 0.35271809, 0.34291354, 0.33916453,
       0.359094  , 0.31584463, 0.40697953, 0.3733516 , 0.3247462 ,
       0.34236452, 0.2928598 , 0.354238  , 0.38166362, 0.35882866,
       0.3782471 , 0.38768637, 0.34457606, 0.41593662, 0.36847502],      dtype=float32)
                },
                'post': {
                    'b': Array([-0.00264038,  0.00694185,  0.01401373, ..., -0.00361281,
       -0.01312758, -0.00154151], dtype=float32),
                    'dora_m': Array([[[0.50927126, 0.47577742, 0.43136907, ..., 0.51659065,
         0.43074188, 0.4861642 ],
        [0.45254996, 0.49526384, 0.44537336, ..., 0.4614066 ,
         0.47868434, 0.4912089 ],
        [0.43411195, 0.4823954 , 0.4769836 , ..., 0.48540527,
         0.47190627, 0.46048462],
        ...,
        [0.45022148, 0.4832163 , 0.49625885, ..., 0.5021365 ,
         0.4634759 , 0.47933272],
        [0.5106275 , 0.52129775, 0.46076703, ..., 0.514461  ,
         0.5036143 , 0.5710525 ],
        [0.45240322, 0.44923005, 0.49739584, ..., 0.4604238 ,
         0.47117162, 0.5109038 ]]], dtype=float32),
                    'lora_a': Array([[-0.10846648],
       [ 0.2945283 ],
       [-0.2603643 ],
       ...,
       [-0.7982759 ],
       [ 0.7376247 ],
       [-0.084032  ]], dtype=float32),
                    'lora_b': Array([[ 7.9751189e-06],
       [-1.5565633e-05],
       [-1.3373924e-05],
       ...,
       [-1.2865105e-05],
       [ 1.7853225e-05],
       [-1.5533302e-05]], dtype=float32),
                    'w': Array([[[ 1.87228695e-02,  2.09279335e-03,  1.22624869e-02, ...,
         -5.55560971e-03,  4.06807842e-04,  6.86809141e-03],
        [-1.01454938e-02, -1.03525901e-02,  1.22757955e-03, ...,
          7.81850191e-04,  1.15038836e-02, -2.10440508e-03],
        [ 5.03648480e-04, -2.61391252e-02,  1.11459903e-02, ...,
          2.29138695e-02, -1.46881621e-02,  3.90026392e-03],
        ...,
        [ 1.56186810e-02,  7.08769402e-03,  8.49680137e-03, ...,
          8.11524130e-03,  1.08613428e-02,  4.19352483e-03],
        [-4.58357064e-03, -2.25086100e-02,  1.24246010e-03, ...,
         -1.20283552e-02, -1.25527671e-02,  1.78278312e-02],
        [-1.64922811e-02, -2.23112721e-02,  1.08939959e-02, ...,
          7.43396394e-03, -7.29675591e-03, -1.77015848e-02]],

...,

[[ 8.73747561e-03, -1.13629596e-02, -7.38181034e-03, ...,
          1.33837434e-02, -9.68668051e-03, -1.46931009e-02],
        [ 6.46871282e-03,  3.81157082e-03,  7.98682682e-03, ...,
         -4.13137209e-03,  1.93525141e-03, -1.31269665e-02],
        [-1.89980771e-02,  7.40556745e-04, -2.40534265e-03, ...,
          8.91357474e-03,  5.45089459e-03, -1.62918661e-02],
        ...,
        [-1.69311017e-02, -2.38751173e-02,  1.52366096e-02, ...,
          1.46889640e-02,  5.74547332e-03, -7.65995064e-05],
        [ 1.58907333e-03, -1.24421241e-02, -1.88286137e-02, ...,
          1.26177752e-02,  2.40578200e-03,  5.15837781e-03],
        [-2.36421563e-02,  2.78542517e-03,  2.30298936e-02, ...,
         -1.15507739e-02, -7.25464104e-03,  2.17000712e-02]]],      dtype=float32)
                },
                'query': {
                    'b': Array([[-0.00262852,  0.00193108, -0.00039411, ..., -0.00492288,
         0.00507819,  0.0045451 ],
       [-0.00436657, -0.01169078,  0.01313771, ..., -0.01245482,
         0.00836836, -0.00160828],
       [-0.00267721, -0.00322388,  0.00058989, ..., -0.00694676,
        -0.012724  , -0.00606337],
       ...,
       [ 0.00056436,  0.01125247, -0.01011393, ..., -0.00623019,
        -0.00752389,  0.00585707],
       [ 0.00551907, -0.00466406,  0.00090617, ..., -0.00812737,
        -0.00139521, -0.00883428],
       [ 0.00855688, -0.00175026, -0.01494024, ...,  0.01107276,
         0.00470326,  0.00555534]], dtype=float32),
                    'dora_m': Array([[[0.7411379 , 0.6959757 , 0.8067702 , ..., 0.74071777,
         0.82333636, 0.70498514],
        [0.63652575, 0.72305924, 0.8372227 , ..., 0.8650049 ,
         0.61539924, 0.6335241 ],
        [0.6442887 , 0.76941574, 0.7169309 , ..., 0.73979235,
         0.725424  , 0.8275418 ],
        ...,
        [0.9203272 , 0.8940063 , 0.9400773 , ..., 0.96792847,
         0.97503275, 1.0120854 ],
        [1.037943  , 0.98727345, 1.0911691 , ..., 1.0971408 ,
         0.88768816, 1.1740992 ],
        [0.8464271 , 0.72978896, 0.7976973 , ..., 0.8155727 ,
         0.8316726 , 0.690989  ]]], dtype=float32),
                    'lora_a': Array([[-0.10845671],
       [ 0.29455054],
       [-0.26038158],
       ...,
       [-0.7982773 ],
       [ 0.7376215 ],
       [-0.08399924]], dtype=float32),
                    'lora_b': Array([[ 1.12132375e-05],
       [ 1.98008420e-05],
       [ 1.72272612e-05],
       ...,
       [-5.52818892e-06],
       [ 2.04665412e-05],
       [-1.94496770e-05]], dtype=float32),
                    'w': Array([[[-2.32145227e-02, -1.80027988e-02, -1.05454139e-02, ...,
          4.62635048e-02, -3.16045135e-02,  1.83681063e-02],
        [-4.12973464e-02, -1.00914892e-02, -2.15370348e-03, ...,
          1.21688470e-02, -6.79347711e-03, -9.77568701e-03],
        [-2.93969270e-02, -5.16810594e-03, -3.31422426e-02, ...,
          4.39999218e-04, -2.24977881e-02,  1.34276571e-02],
        ...,
        [ 1.30915800e-02,  2.56901397e-03,  1.60468332e-02, ...,
          5.56441862e-03,  2.95666195e-02,  1.05255062e-03],
        [-2.43136138e-02, -1.78905595e-02,  3.00597725e-03, ...,
          1.19134772e-03,  6.64961059e-03, -1.39747770e-03],
        [-1.76110156e-02, -1.31995883e-03, -6.59585185e-03, ...,
          5.34036756e-03,  1.04377223e-02,  1.93941332e-02]],

...,

[[-8.00139830e-03,  1.62869226e-02, -2.50888988e-02, ...,
          1.60501990e-02,  1.23645673e-02,  4.42911731e-03],
        [-1.40104657e-02,  5.45975715e-02,  2.15243883e-02, ...,
         -2.07017269e-02, -5.22241881e-03, -1.60436239e-02],
        [ 1.52988229e-02,  4.63366173e-02,  2.69267075e-02, ...,
         -7.67846312e-03,  1.13997133e-02,  5.20083569e-02],
        ...,
        [ 4.15044986e-02,  6.75549870e-03,  5.89986425e-03, ...,
          3.34248096e-02, -2.61541680e-02, -8.67828447e-03],
        [-4.17082123e-02,  1.01776039e-02, -4.68776189e-02, ...,
          1.82378702e-02,  3.92180420e-02, -6.68293834e-02],
        [-5.26580913e-03,  3.55641334e-03, -4.06127376e-03, ...,
          8.88812472e-04,  4.26395908e-02,  3.76701988e-02]]],      dtype=float32)
                },
                'value': {
                    'b': Array([[ 4.3092333e-03, -1.0670313e-03,  1.0951947e-03, ...,
        -7.5847393e-06, -2.9846833e-03, -2.8228675e-05],
       [ 1.0933492e-03,  3.7175193e-03, -1.4860635e-03, ...,
         1.9951609e-03, -3.4172609e-03,  4.1828798e-03],
       [ 1.5554003e-03, -3.2844695e-03, -4.2517087e-03, ...,
        -6.4385339e-04, -2.6835120e-04, -5.1554147e-04],
       ...,
       [-5.4495255e-03, -4.6499642e-03, -3.1639240e-03, ...,
        -2.4868713e-03, -5.6621089e-04, -2.1520587e-05],
       [-1.2494889e-03,  4.1477205e-03, -2.2556106e-04, ...,
        -4.7293715e-03, -1.6261863e-03, -1.2881723e-03],
       [ 4.9109582e-04, -1.3274476e-03,  1.4091528e-03, ...,
        -1.3600906e-03,  5.9296563e-03,  8.0530928e-04]], dtype=float32),
                    'dora_m': Array([[[0.52868706, 0.4588456 , 0.4360924 , ..., 0.5056494 ,
         0.44141078, 0.48564065],
        [0.42356193, 0.49531782, 0.42251417, ..., 0.45007628,
         0.47780603, 0.48437282],
        [0.4575237 , 0.46944562, 0.49771714, ..., 0.4750855 ,
         0.42793   , 0.46321765],
        ...,
        [0.43922392, 0.4659978 , 0.4922388 , ..., 0.4917019 ,
         0.47714734, 0.47792545],
        [0.51889265, 0.5586302 , 0.4438029 , ..., 0.53018326,
         0.53596026, 0.62285095],
        [0.44028997, 0.44600555, 0.51922464, ..., 0.47069794,
         0.49034953, 0.5116235 ]]], dtype=float32),
                    'lora_a': Array([[-0.10848485],
       [ 0.2945549 ],
       [-0.26035795],
       ...,
       [-0.79827935],
       [ 0.737614  ],
       [-0.08399877]], dtype=float32),
                    'lora_b': Array([[-1.0781089e-06],
       [ 1.9513400e-05],
       [ 1.8644892e-05],
       ...,
       [ 1.8479303e-05],
       [-2.0403801e-05],
       [ 1.9073743e-05]], dtype=float32),
                    'w': Array([[[-2.69760992e-02,  1.25788096e-02, -7.61376414e-03, ...,
         -1.52490120e-02,  5.33550233e-03,  7.93699082e-03],
        [ 1.05836606e-02,  1.82690602e-02,  1.03190932e-02, ...,
         -1.38023579e-02,  1.13194250e-02, -1.86773445e-02],
        [ 6.18274929e-03,  2.25341134e-02, -2.55283359e-02, ...,
          9.22846049e-03,  2.76181158e-02, -8.00884049e-03],
        ...,
        [-2.94117315e-04, -3.88901588e-03,  1.62602384e-02, ...,
          6.27086684e-03, -2.42737625e-02,  1.65726133e-02],
        [-2.14569233e-02,  4.66705300e-03,  7.93393422e-03, ...,
          2.82598324e-02,  1.51329385e-02, -9.63309221e-03],
        [ 1.53947799e-02, -1.65367703e-04,  2.37261318e-03, ...,
          5.47492637e-05,  1.14596752e-03, -3.79634369e-03]],

...,

[[ 1.12230079e-02,  4.17726627e-03, -1.50599014e-02, ...,
         -1.54130748e-02,  4.49147914e-03,  2.02064924e-02],
        [ 7.14984071e-03, -1.13470023e-02, -7.55088637e-03, ...,
          2.19306722e-03, -5.27518662e-03,  1.26894163e-02],
        [ 1.88901164e-02, -2.14834549e-02, -5.00168302e-04, ...,
         -1.98321044e-02,  2.19250051e-03,  1.37647400e-02],
        ...,
        [ 8.68065562e-03,  1.22577073e-02, -1.24850739e-02, ...,
         -7.33076176e-03, -2.36104825e-03,  6.77861832e-03],
        [ 1.20055564e-02, -7.20065180e-03,  1.14093032e-02, ...,
         -1.32740131e-02, -2.68567689e-02, -3.12830806e-02],
        [ 8.80962703e-03, -2.94224042e-02, -1.45735533e-03, ...,
         -2.19043531e-02,  1.76994188e-04,  3.11435293e-03]]],      dtype=float32)
                }
            }
        },
        'x_layers_16': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': Array([-0.01093779, -0.006601  , -0.01157581, ..., -0.00636036,
        0.00338994, -0.00561759], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.1466405 , 1.0764251 , 1.13969   , ..., 1.1958363 , 0.94589967,
        1.0283298 ]], dtype=float32),
                        'lora_a': Array([[-0.10845026],
       [ 0.29455128],
       [-0.26035973],
       ...,
       [-0.798237  ],
       [ 0.7376293 ],
       [-0.08399849]], dtype=float32),
                        'lora_b': Array([[-6.1688061e-06],
       [ 1.7766430e-05],
       [ 1.2769769e-05],
       ...,
       [ 0.0000000e+00],
       [ 2.0219542e-05],
       [ 1.6517175e-05]], dtype=float32),
                        'w': Array([[-0.07065611,  0.0366316 ,  0.00286838, ...,  0.04645369,
        -0.00233936, -0.00317006],
       [ 0.01336469,  0.00203272, -0.04045084, ...,  0.00291746,
        -0.02290988, -0.0063614 ],
       [-0.00353894,  0.0236262 , -0.0291876 , ..., -0.01251738,
        -0.01119486,  0.0026152 ],
       ...,
       [-0.00828904,  0.0029501 ,  0.00390786, ...,  0.02516262,
        -0.01356525,  0.00672958],
       [-0.00160194, -0.02658143,  0.05103948, ...,  0.01940908,
        -0.03386628,  0.02277668],
       [-0.00529792,  0.04206442,  0.01578536, ..., -0.01186924,
         0.02141595,  0.01679806]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': Array([-0.00202641,  0.00375642,  0.00326318, ..., -0.00049565,
       -0.00301311,  0.00015281], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.2694132, 1.3006377, 1.2394526, ..., 1.3128942, 1.2633698,
        1.2886333]], dtype=float32),
                        'lora_a': Array([[-0.10847995],
       [ 0.2945277 ],
       [-0.26037923],
       ...,
       [-0.79826015],
       [ 0.7376188 ],
       [-0.08403596]], dtype=float32),
                        'lora_b': Array([[ 5.9668669e-06],
       [ 1.7471957e-05],
       [ 1.2054656e-07],
       ...,
       [ 2.0826610e-05],
       [ 1.5376285e-05],
       [-1.0675214e-05]], dtype=float32),
                        'w': Array([[-0.00584749, -0.01089616, -0.05799555, ..., -0.03713508,
         0.00729555, -0.01389926],
       [-0.04660178, -0.027352  , -0.04065514, ...,  0.03577564,
        -0.03868281,  0.00700549],
       [-0.03919972, -0.01063161,  0.0586987 , ...,  0.01688408,
         0.04164346, -0.01884241],
       ...,
       [ 0.02697367, -0.01475049,  0.04031842, ..., -0.04789248,
        -0.02740616,  0.0376101 ],
       [-0.03518107,  0.0076696 ,  0.02464686, ..., -0.01843506,
         0.00698606, -0.04271198],
       [ 0.05981452,  0.03818025, -0.0107589 , ..., -0.01857778,
        -0.00216738,  0.00547846]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': Array([ 0.00151769,  0.00547054,  0.01455967, ..., -0.0095241 ,
       -0.02066464, -0.00408657], dtype=float32),
                    'scale': Array([ 0.00557752, -0.0064744 ,  0.02385767, ..., -0.0015815 ,
        0.0141864 ,  0.02011438], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': Array([1.1342393, 1.0935222, 1.1052631, ..., 1.1164135, 1.0588753,
       1.131423 ], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': Array([[-2.5851612e-03,  1.8234068e-03,  1.4820995e-03, ...,
        -1.1297370e-03, -1.0774004e-03, -8.2510058e-05],
       [-1.3637317e-03, -1.5217112e-03, -2.3586715e-04, ...,
         7.5874908e-04,  1.4486602e-03,  1.9991904e-04],
       [-1.3591145e-03, -8.1638672e-04, -3.9848150e-03, ...,
        -2.2466586e-03, -9.1814104e-04,  5.3033553e-04],
       ...,
       [ 4.4141963e-04,  1.4761785e-03,  3.0662501e-04, ...,
         1.2143034e-03, -4.2294950e-04, -3.1919666e-03],
       [-3.9186152e-03,  1.6968115e-03, -2.6914002e-03, ...,
        -2.7568685e-03, -4.9815909e-03, -5.5790623e-04],
       [ 5.4388520e-06,  1.6627987e-03,  1.9495675e-03, ...,
        -1.2228332e-03,  3.4314577e-04, -1.3438117e-03]], dtype=float32),
                    'dora_m': Array([[[0.43714982, 0.4283233 , 0.40480605, ..., 0.6134639 ,
         0.5275718 , 0.56580323],
        [0.6734948 , 0.7917611 , 0.5761828 , ..., 0.6602124 ,
         0.54075664, 0.6867239 ],
        [0.49247903, 0.47324133, 0.51652795, ..., 0.4879602 ,
         0.4562971 , 0.5633198 ],
        ...,
        [0.56665045, 0.5098786 , 0.48794445, ..., 0.59680676,
         0.4615795 , 0.37047753],
        [0.47106877, 0.4855661 , 0.6218938 , ..., 0.55184895,
         0.48305753, 0.48108295],
        [0.597833  , 0.5316523 , 0.50892776, ..., 0.5161282 ,
         0.6191833 , 0.498586  ]]], dtype=float32),
                    'lora_a': Array([[-0.10847602],
       [ 0.29451308],
       [-0.2603853 ],
       ...,
       [-0.79824466],
       [ 0.7376466 ],
       [-0.08403572]], dtype=float32),
                    'lora_b': Array([[ 1.1920699e-05],
       [ 8.1739117e-06],
       [ 9.3355384e-06],
       ...,
       [-1.3567088e-05],
       [-2.7543440e-06],
       [-1.5085335e-05]], dtype=float32),
                    'w': Array([[[ 1.42219989e-02,  2.71342508e-03,  1.42611526e-02, ...,
         -6.63716695e-04,  2.06843554e-03, -1.18770655e-02],
        [-1.26144430e-02, -2.13406812e-02,  1.53633235e-02, ...,
         -2.08577700e-02,  5.63759264e-03,  3.21701840e-02],
        [ 9.53059178e-03, -3.01317032e-02,  2.32374743e-02, ...,
         -4.51727508e-04, -9.19227581e-03, -3.94389872e-03],
        ...,
        [-2.69971937e-02, -6.48251968e-03,  4.16581379e-03, ...,
          7.08622346e-03,  2.02608900e-03, -1.23441136e-02],
        [-1.16614746e-02,  2.02821363e-02, -9.51786153e-03, ...,
          1.27844163e-03,  3.91502958e-03, -2.33424399e-02],
        [-1.08019337e-02, -1.44250272e-02, -1.16862142e-02, ...,
          1.82441156e-02, -1.74487494e-02,  2.79446272e-03]],

...,

[[-6.31908793e-03,  6.94034388e-03,  7.63501693e-03, ...,
         -1.41769815e-02, -6.49016025e-03, -6.64137816e-03],
        [ 2.29326990e-02, -1.21046333e-02, -2.68131159e-02, ...,
         -1.35539537e-02, -3.14271003e-02,  2.52824724e-02],
        [-6.18695933e-03,  4.36464651e-03, -2.61276700e-02, ...,
         -1.36762187e-02, -7.50785926e-03,  4.96061035e-02],
        ...,
        [ 2.64439005e-02,  8.17085337e-03, -2.24624947e-03, ...,
         -4.54828292e-02,  5.85463829e-03,  1.52172390e-02],
        [-4.02525906e-03, -4.76133544e-03,  1.62002854e-02, ...,
          4.21542348e-03,  2.11307444e-02, -2.67693736e-02],
        [-1.22092944e-02,  1.79102104e-02, -1.53264159e-03, ...,
          1.92070529e-02, -1.04104066e-02,  6.43276563e-03]]],      dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': Array([0.34839326, 0.34487322, 0.30048433, 0.40271607, 0.36474398,
       0.37258643, 0.33030462, 0.32204038, 0.2811675 , 0.3941069 ,
       0.40174028, 0.31793132, 0.34787026, 0.35276464, 0.36014393,
       0.38798308, 0.36074895, 0.36447507, 0.35500136, 0.38915566,
       0.38850445, 0.37506366, 0.3692204 , 0.37950742, 0.39861467,
       0.387693  , 0.40703672, 0.35293734, 0.40224758, 0.34825236,
       0.3776654 , 0.39786416, 0.34274727, 0.34450045, 0.45802835,
       0.3469762 , 0.40184072, 0.3448745 , 0.44639596, 0.37119558,
       0.3595001 , 0.34055847, 0.34886846, 0.32589296, 0.4076769 ,
       0.2716943 , 0.36786398, 0.3192062 , 0.401973  , 0.3555044 ,
       0.4049436 , 0.36825705, 0.3393343 , 0.3791553 , 0.39327073,
       0.37439272, 0.37110728, 0.28624743, 0.36308977, 0.37162504,
       0.35790285, 0.3320287 , 0.34310248, 0.3696956 , 0.36224914,
       0.36394373, 0.3157535 , 0.3760167 , 0.38727936, 0.3700242 ,
       0.35368103, 0.39586368, 0.31667173, 0.32127523, 0.41776994,
       0.35321888, 0.35742685, 0.33300877, 0.39574778, 0.36631098],      dtype=float32)
                },
                'post': {
                    'b': Array([-0.00294997,  0.00812847,  0.0093169 , ..., -0.0007974 ,
       -0.01477665, -0.0010464 ], dtype=float32),
                    'dora_m': Array([[[0.37776497, 0.36699834, 0.49360266, ..., 0.4880904 ,
         0.5159852 , 0.44437167],
        [0.4524206 , 0.50380814, 0.5202612 , ..., 0.5547338 ,
         0.5010504 , 0.5114829 ],
        [0.4249985 , 0.49509868, 0.48645118, ..., 0.40940145,
         0.4712107 , 0.46862563],
        ...,
        [0.417384  , 0.4215999 , 0.32868445, ..., 0.4338313 ,
         0.4866224 , 0.46670195],
        [0.5006541 , 0.52115965, 0.5077461 , ..., 0.36015883,
         0.5024391 , 0.45858997],
        [0.45918372, 0.48157915, 0.46235454, ..., 0.4969467 ,
         0.52391714, 0.4578652 ]]], dtype=float32),
                    'lora_a': Array([[-0.10847276],
       [ 0.29453102],
       [-0.26035327],
       ...,
       [-0.7982679 ],
       [ 0.7376349 ],
       [-0.08402448]], dtype=float32),
                    'lora_b': Array([[-1.3419605e-05],
       [-1.9924184e-05],
       [-3.6008714e-07],
       ...,
       [-1.6714535e-05],
       [-4.7943176e-06],
       [ 2.0199976e-05]], dtype=float32),
                    'w': Array([[[-6.26317598e-03, -4.82488278e-04, -8.57751351e-03, ...,
         -2.05452647e-02,  2.42175907e-02, -6.70188665e-03],
        [ 1.41657665e-02, -6.73066778e-03, -1.66489817e-02, ...,
         -1.62274856e-02,  2.53450533e-04, -9.73656075e-04],
        [ 1.20002348e-02,  5.99002978e-03, -2.94627491e-02, ...,
         -3.66344047e-03, -1.12276413e-02,  2.94008339e-03],
        ...,
        [-7.90430978e-03,  8.99317674e-03, -1.75724679e-03, ...,
         -8.32430273e-03,  2.25645519e-04,  5.43238735e-03],
        [-1.56619363e-02, -8.48766067e-04, -1.67979728e-02, ...,
          7.18162162e-03, -7.05741858e-03,  1.16398856e-02],
        [-1.49521208e-03, -9.60188918e-03, -1.99567042e-02, ...,
         -8.80062301e-03,  1.89761408e-02,  7.87581783e-03]],

...,

[[-1.23030320e-02,  2.26234575e-03,  1.09223975e-03, ...,
          3.52696120e-03,  1.31098898e-02, -3.36450264e-02],
        [ 7.17036705e-03, -4.04133112e-04,  1.35272148e-03, ...,
          9.51828528e-03,  4.72732633e-03, -9.87181440e-03],
        [-3.73343774e-03, -6.48180163e-03, -1.70778241e-02, ...,
         -6.57120021e-03,  2.25233845e-02, -3.19988653e-03],
        ...,
        [ 1.78108849e-02,  2.42996644e-02, -1.22569716e-02, ...,
         -3.13344225e-02, -1.28056565e-02, -6.51384518e-03],
        [ 2.63769217e-02, -2.26650685e-02, -8.46231449e-03, ...,
         -9.00840573e-03,  2.37338874e-03,  1.78988148e-02],
        [ 2.86986656e-03, -1.65359688e-03,  2.27783080e-02, ...,
         -4.38299682e-03,  2.24830187e-03, -3.57659650e-04]]],      dtype=float32)
                },
                'query': {
                    'b': Array([[-0.01165662,  0.02085786,  0.02257565, ...,  0.00330295,
        -0.00433043, -0.00432216],
       [ 0.0011895 ,  0.00280689, -0.00527828, ..., -0.00170289,
         0.00037589, -0.00220599],
       [-0.00748594, -0.01677281, -0.00804976, ..., -0.01231432,
        -0.01559735,  0.01375802],
       ...,
       [-0.01084539, -0.00508603,  0.01091589, ..., -0.00500977,
        -0.01142354,  0.01658001],
       [-0.00247246,  0.0003816 ,  0.00308422, ..., -0.00597533,
        -0.00084152, -0.0063736 ],
       [ 0.00854591, -0.01875939, -0.01584159, ...,  0.01792025,
        -0.00124592,  0.01076551]], dtype=float32),
                    'dora_m': Array([[[0.6945532 , 0.8499065 , 0.8057483 , ..., 0.6025619 ,
         0.73453057, 0.69803715],
        [0.9383713 , 0.76910335, 0.92195475, ..., 0.890208  ,
         1.0819623 , 0.79149836],
        [0.5647549 , 0.7690336 , 0.8210626 , ..., 0.7471158 ,
         0.7518307 , 0.70141333],
        ...,
        [0.80820584, 0.7574888 , 0.7447874 , ..., 0.7828458 ,
         0.63042223, 0.66209966],
        [0.82018524, 0.9491207 , 0.82454914, ..., 0.7993645 ,
         1.1135318 , 0.89621747],
        [0.8031563 , 0.8306203 , 0.7685996 , ..., 0.8110916 ,
         0.76028126, 0.86419594]]], dtype=float32),
                    'lora_a': Array([[-0.1084493 ],
       [ 0.29455242],
       [-0.2603813 ],
       ...,
       [-0.7982711 ],
       [ 0.73761994],
       [-0.08399883]], dtype=float32),
                    'lora_b': Array([[-8.5825568e-06],
       [-2.0571690e-05],
       [-2.0369220e-05],
       ...,
       [-1.5063687e-05],
       [ 1.8546443e-05],
       [-1.7412947e-05]], dtype=float32),
                    'w': Array([[[-3.76275554e-02, -7.15244235e-03,  3.47784422e-02, ...,
          4.98246141e-02, -4.11462523e-02, -1.32022621e-02],
        [-2.34063361e-02, -1.19711217e-02, -2.04937831e-02, ...,
          1.11456187e-02,  2.34308112e-02, -2.43430343e-02],
        [-9.23752226e-03,  1.40757095e-02, -1.59042459e-02, ...,
          1.05783297e-03, -1.82259232e-02, -8.06174055e-03],
        ...,
        [ 1.27868764e-02,  1.28190068e-03, -4.98180129e-02, ...,
         -4.39898856e-02,  6.19128020e-03,  6.68785395e-03],
        [-5.87123586e-03, -2.46423464e-02,  2.46517453e-02, ...,
          2.01327037e-02, -3.63838188e-02,  1.02107450e-02],
        [-1.71772437e-03,  6.64134882e-03, -1.76312551e-02, ...,
          5.36030717e-03, -1.41256815e-03, -1.02146827e-02]],

...,

[[ 6.06435612e-02,  2.95455698e-02,  1.51243405e-02, ...,
          3.53831775e-03,  9.64399520e-03, -1.09638572e-02],
        [-4.68350127e-02,  3.44929323e-02,  2.58492399e-03, ...,
          8.91290884e-03, -2.92156115e-02,  3.20116431e-02],
        [ 3.71786840e-02, -3.70884401e-04,  1.91319417e-02, ...,
          1.18764145e-02,  5.01254341e-03,  5.30658336e-03],
        ...,
        [-3.02066002e-03, -1.16653452e-02, -3.25030982e-02, ...,
         -9.35567636e-03,  1.97983272e-02,  2.31254101e-02],
        [-3.90016064e-02, -2.88063362e-02, -9.27522033e-03, ...,
         -1.25017455e-02, -1.60281304e-02, -1.13976942e-02],
        [ 3.40925641e-02, -1.13470471e-02, -3.17137106e-03, ...,
         -5.07070217e-03,  1.89429813e-03,  3.31671350e-02]]],      dtype=float32)
                },
                'value': {
                    'b': Array([[ 2.3536319e-03, -5.8421714e-04,  2.6541166e-03, ...,
        -1.1526708e-03,  1.2811483e-03, -6.3673397e-05],
       [-3.6600938e-03, -3.3384843e-03, -1.1473114e-03, ...,
        -1.4382317e-03, -3.8280077e-03,  5.3748143e-05],
       [-6.2878610e-04, -2.3630157e-03, -1.6891264e-03, ...,
        -4.5074243e-03,  2.0703450e-03,  1.6706111e-03],
       ...,
       [ 1.0337565e-03, -6.6874768e-03,  7.8707105e-03, ...,
         3.8281935e-03,  2.7246573e-03, -5.4889736e-03],
       [ 4.5474949e-03,  1.8496951e-03, -4.7515641e-04, ...,
         1.6249692e-03, -2.1909401e-03, -2.1236469e-03],
       [ 6.4964488e-04,  1.0762867e-03, -3.3429137e-03, ...,
         2.1054570e-03,  3.9049805e-04, -2.0609915e-03]], dtype=float32),
                    'dora_m': Array([[[0.3692356 , 0.35154346, 0.48467737, ..., 0.49891016,
         0.47129455, 0.42752388],
        [0.46169922, 0.5203268 , 0.5222628 , ..., 0.5284995 ,
         0.4929168 , 0.5160797 ],
        [0.42671493, 0.49103582, 0.49274555, ..., 0.4043901 ,
         0.5015137 , 0.46594763],
        ...,
        [0.43133202, 0.41935527, 0.34410506, ..., 0.44367713,
         0.46297202, 0.4869781 ],
        [0.51341224, 0.502494  , 0.49934757, ..., 0.35287842,
         0.4880844 , 0.4320805 ],
        [0.44179216, 0.4690609 , 0.46626306, ..., 0.47839445,
         0.49141127, 0.4712712 ]]], dtype=float32),
                    'lora_a': Array([[-0.10844902],
       [ 0.2945547 ],
       [-0.2603624 ],
       ...,
       [-0.79827684],
       [ 0.7376156 ],
       [-0.08399867]], dtype=float32),
                    'lora_b': Array([[-1.9495556e-05],
       [-8.4905541e-06],
       [ 1.3115616e-05],
       ...,
       [-1.0996387e-05],
       [ 2.3841883e-06],
       [-5.5244204e-06]], dtype=float32),
                    'w': Array([[[-2.76697311e-03,  9.07628099e-04,  2.88689364e-04, ...,
          4.36796620e-03,  2.63751904e-03, -1.84235796e-02],
        [ 8.94250628e-03, -3.43922116e-02,  1.04978001e-02, ...,
          7.22295837e-03, -8.83479882e-03,  9.46309417e-03],
        [ 9.88198258e-03, -3.21986005e-02, -1.89686548e-02, ...,
         -1.78769864e-02, -1.25460494e-02,  1.63281579e-02],
        ...,
        [-1.66412315e-03, -8.80832225e-03,  1.40606370e-02, ...,
          3.45968977e-02,  4.83957585e-03,  1.87827908e-02],
        [ 4.24764538e-03, -2.04535387e-02,  2.49281321e-02, ...,
         -1.04700867e-02,  2.57115997e-02, -8.19846988e-03],
        [-1.17270052e-02, -9.50942002e-03, -2.19905041e-02, ...,
          2.44821776e-02, -7.87706580e-03, -1.02372207e-02]],

...,

[[ 2.47591967e-03, -2.99220229e-03,  4.34108218e-03, ...,
          6.08616602e-03,  7.70103559e-03,  4.92395193e-04],
        [-2.45214876e-04, -4.27352125e-03,  2.37497184e-02, ...,
          1.16175665e-02, -8.89002532e-03,  1.17512504e-02],
        [ 5.56151150e-03, -2.24482492e-02,  4.73437086e-03, ...,
          6.34752354e-03,  1.10943178e-02,  3.15959100e-03],
        ...,
        [ 1.53137231e-02, -2.43450031e-02,  2.87173828e-03, ...,
          3.01783346e-02,  1.53354611e-02,  6.15234673e-03],
        [-1.02614937e-02,  1.27734924e-02,  9.20217112e-03, ...,
         -1.02896625e-02,  1.91032458e-02,  2.17168345e-05],
        [-1.01738321e-02, -6.64736098e-03, -4.09644879e-02, ...,
          9.67943482e-03,  1.93070676e-02,  9.33439843e-03]]],      dtype=float32)
                }
            }
        },
        'x_layers_17': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': Array([ 0.00171706, -0.00131204, -0.0021915 , ..., -0.00766173,
        0.00270988, -0.00303804], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[0.99976784, 0.9552299 , 1.1004167 , ..., 1.0892789 , 0.9593518 ,
        0.95226973]], dtype=float32),
                        'lora_a': Array([[-0.10844932],
       [ 0.29455438],
       [-0.26035058],
       ...,
       [-0.7982384 ],
       [ 0.73761487],
       [-0.08399893]], dtype=float32),
                        'lora_b': Array([[ 1.7743292e-05],
       [ 5.7058114e-06],
       [-7.1381392e-06],
       ...,
       [ 1.8197814e-05],
       [-1.3812807e-05],
       [-1.8585473e-05]], dtype=float32),
                        'w': Array([[ 0.02472598,  0.00262219,  0.05628514, ..., -0.04744835,
        -0.05583418,  0.000448  ],
       [ 0.0435979 , -0.03077106, -0.01843363, ..., -0.03928912,
         0.00057252, -0.00394176],
       [ 0.02986974, -0.03444536, -0.03493382, ...,  0.0275128 ,
         0.00404201, -0.03031758],
       ...,
       [-0.01906034,  0.00683718, -0.02937934, ...,  0.02432913,
        -0.02658359, -0.02528914],
       [ 0.02618262, -0.03669364,  0.0492231 , ..., -0.0325532 ,
        -0.00535654,  0.03550595],
       [ 0.01585514,  0.01979536,  0.02232511, ..., -0.00242681,
        -0.02909306, -0.05983097]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': Array([-0.00295017,  0.00484709,  0.00511358, ..., -0.00108915,
       -0.00493497, -0.00364521], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.3913206, 1.324125 , 1.2769568, ..., 1.3841733, 1.4442608,
        1.4156507]], dtype=float32),
                        'lora_a': Array([[-0.10845184],
       [ 0.29455462],
       [-0.26037642],
       ...,
       [-0.79824424],
       [ 0.73765695],
       [-0.08399822]], dtype=float32),
                        'lora_b': Array([[ 2.2163878e-05],
       [ 1.7892140e-05],
       [-5.6806762e-06],
       ...,
       [-2.1718080e-05],
       [-1.8644161e-05],
       [-8.6559166e-06]], dtype=float32),
                        'w': Array([[ 0.00931492, -0.01786293,  0.02748417, ..., -0.0600135 ,
        -0.01249322,  0.02373556],
       [ 0.03530489,  0.00356561,  0.04185544, ..., -0.02650248,
        -0.0146055 , -0.05268423],
       [ 0.03967872, -0.0358183 ,  0.0471034 , ..., -0.03817839,
         0.02537379, -0.06090042],
       ...,
       [ 0.0485337 ,  0.00673793, -0.02475552, ..., -0.01539302,
         0.01099681,  0.01217936],
       [ 0.03491158,  0.03560209,  0.0085284 , ..., -0.01987701,
         0.02958684,  0.03682547],
       [-0.04517267, -0.02559242,  0.04155496, ..., -0.03713605,
        -0.01925383,  0.02169372]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': Array([ 0.0058205 , -0.00151459,  0.0105388 , ..., -0.00131459,
       -0.02444017,  0.00267917], dtype=float32),
                    'scale': Array([0.01300202, 0.02647691, 0.02123367, ..., 0.01254215, 0.06005605,
       0.01016154], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': Array([1.1104951, 1.1179245, 1.0624976, ..., 1.1053315, 1.0585247,
       1.0821815], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': Array([[ 0.00256548, -0.00106355,  0.00024083, ..., -0.00201515,
         0.00219756, -0.00327231],
       [ 0.00121145, -0.00157773,  0.00120874, ..., -0.00152556,
         0.00318669, -0.00010758],
       [-0.00213632,  0.00069429,  0.00132417, ...,  0.00021959,
        -0.00080004, -0.0014997 ],
       ...,
       [-0.0032619 , -0.00237538,  0.00118759, ...,  0.00123081,
         0.00139293, -0.00295219],
       [ 0.00184485,  0.000301  ,  0.00165043, ..., -0.00173555,
         0.00077969,  0.00220971],
       [ 0.000826  ,  0.00090565, -0.00069976, ..., -0.00110533,
        -0.00024552, -0.00129298]], dtype=float32),
                    'dora_m': Array([[[0.54827756, 0.5254321 , 0.57318544, ..., 0.5439552 ,
         0.57984436, 0.54243875],
        [0.69621253, 0.6518897 , 0.53641635, ..., 0.58533484,
         0.59114546, 0.53628707],
        [0.48981088, 0.5164488 , 0.5058307 , ..., 0.44774124,
         0.44726235, 0.4628996 ],
        ...,
        [0.40840924, 0.51653093, 0.559193  , ..., 0.48497418,
         0.4670829 , 0.543182  ],
        [0.4809209 , 0.5127862 , 0.4932278 , ..., 0.5848788 ,
         0.57917845, 0.55235356],
        [0.5056341 , 0.6872438 , 0.4948368 , ..., 0.58865184,
         0.43810496, 0.5983575 ]]], dtype=float32),
                    'lora_a': Array([[-0.10849038],
       [ 0.2945176 ],
       [-0.26037714],
       ...,
       [-0.7982477 ],
       [ 0.7376395 ],
       [-0.08402278]], dtype=float32),
                    'lora_b': Array([[ 1.7277511e-05],
       [ 1.0772973e-05],
       [-2.1113125e-05],
       ...,
       [-7.7180484e-06],
       [-2.1874084e-05],
       [-1.6725162e-05]], dtype=float32),
                    'w': Array([[[ 0.00062362, -0.01420906,  0.00070949, ...,  0.01102791,
          0.01875118, -0.00241041],
        [-0.01302008, -0.01643992, -0.00820376, ..., -0.01269134,
         -0.01051504, -0.00433114],
        [ 0.01605858,  0.03422265, -0.01969025, ..., -0.02132047,
         -0.01710039, -0.00646175],
        ...,
        [-0.01284659, -0.027529  , -0.009594  , ..., -0.02514833,
          0.00789621, -0.02574673],
        [-0.01678951, -0.00012382, -0.00577383, ..., -0.00447377,
          0.01541107,  0.03425353],
        [-0.00548437, -0.00744928, -0.0070674 , ...,  0.02867692,
          0.0161877 ,  0.00638222]],

...,

[[ 0.00251885,  0.0151092 , -0.00846976, ...,  0.02188397,
         -0.00930506,  0.01001358],
        [-0.0057018 , -0.01241554,  0.00730267, ...,  0.02563364,
          0.01692494,  0.00170398],
        [ 0.01367783,  0.00302956, -0.00021606, ..., -0.00508406,
         -0.01428949, -0.02000578],
        ...,
        [ 0.00197864, -0.0168051 , -0.00532901, ..., -0.00698534,
          0.00512153,  0.00303917],
        [-0.00151376, -0.00486665, -0.01476791, ..., -0.00450136,
          0.00636599, -0.01712616],
        [ 0.00871773, -0.00984234, -0.01420589, ..., -0.00379258,
          0.00297406, -0.01527084]]], dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': Array([0.18329173, 0.306529  , 0.33626303, 0.34032774, 0.31197897,
       0.26566285, 0.21109979, 0.2993935 , 0.3012725 , 0.13591361,
       0.25008982, 0.17057759, 0.33780378, 0.19163084, 0.16780691,
       0.3133621 , 0.29482302, 0.29536977, 0.27895793, 0.25672314,
       0.29341793, 0.32210267, 0.29777268, 0.18360291, 0.3179472 ,
       0.29114604, 0.3051625 , 0.25520694, 0.21351007, 0.30364794,
       0.27713385, 0.32244632, 0.27059892, 0.17526585, 0.2978192 ,
       0.26505676, 0.31563473, 0.22453122, 0.2727974 , 0.1579615 ,
       0.14100088, 0.31131014, 0.22335987, 0.28845438, 0.23881653,
       0.25362653, 0.3082384 , 0.2262864 , 0.2552006 , 0.24002545,
       0.18481648, 0.2526903 , 0.22395182, 0.1485628 , 0.33651558,
       0.24711472, 0.2915276 , 0.25388318, 0.27354527, 0.25630867,
       0.21607193, 0.30499017, 0.2570673 , 0.2858135 , 0.21992019,
       0.28136933, 0.16238771, 0.28450722, 0.22485426, 0.33766568,
       0.15231392, 0.25662956, 0.26807204, 0.29038495, 0.3233309 ,
       0.17098823, 0.33013552, 0.27603385, 0.27568522, 0.31669253],      dtype=float32)
                },
                'post': {
                    'b': Array([-0.00257743,  0.00719224,  0.01381382, ...,  0.00106568,
       -0.01411214, -0.0005999 ], dtype=float32),
                    'dora_m': Array([[[0.41278645, 0.33189508, 0.3638787 , ..., 0.5233395 ,
         0.5465488 , 0.43902203],
        [0.37939492, 0.45820406, 0.53179866, ..., 0.46177858,
         0.6444246 , 0.45323   ],
        [0.34980616, 0.4748047 , 0.35920286, ..., 0.44795913,
         0.34815004, 0.4603246 ],
        ...,
        [0.47262368, 0.47228533, 0.41194052, ..., 0.36563542,
         0.3318827 , 0.34748024],
        [0.49956805, 0.54321545, 0.3836747 , ..., 0.45563143,
         0.3434872 , 0.53263426],
        [0.5062811 , 0.48244047, 0.40824533, ..., 0.51357925,
         0.49504173, 0.48512486]]], dtype=float32),
                    'lora_a': Array([[-0.10848937],
       [ 0.29453176],
       [-0.26035446],
       ...,
       [-0.7982763 ],
       [ 0.7376561 ],
       [-0.08400738]], dtype=float32),
                    'lora_b': Array([[ 1.7272105e-05],
       [-1.7438810e-05],
       [ 1.6833532e-05],
       ...,
       [ 1.1349550e-07],
       [-1.6322948e-06],
       [ 6.1000956e-06]], dtype=float32),
                    'w': Array([[[ 2.23880429e-02,  1.80971122e-03,  4.94497735e-03, ...,
          7.46185472e-03, -1.91377718e-02,  5.51110785e-03],
        [-9.65329073e-03, -2.50214022e-02,  6.51374366e-03, ...,
          7.30558007e-04, -1.10678503e-03,  1.15562894e-03],
        [-1.30426968e-02, -1.25828469e-02, -8.35072715e-03, ...,
         -5.25994552e-03, -2.06049960e-02,  1.01154838e-02],
        ...,
        [-3.40172695e-03,  1.04015395e-02,  2.16603074e-02, ...,
          1.59501017e-03, -9.02491342e-03, -4.56762221e-03],
        [ 1.19904121e-02, -1.82692781e-02, -1.07434206e-03, ...,
         -7.59373792e-03,  4.29450534e-03, -4.55015711e-03],
        [ 1.28748650e-02, -6.32785633e-03,  1.58232097e-02, ...,
         -1.22512644e-02,  2.13666223e-02,  3.45551148e-02]],

...,

[[-5.59269032e-03,  9.68667027e-03,  1.00202777e-03, ...,
         -3.58658191e-03,  1.04861651e-02,  4.70099360e-04],
        [ 9.24368761e-03, -7.87344668e-03,  8.12148664e-06, ...,
          9.53086000e-03, -3.83773400e-03,  2.40035001e-02],
        [-1.04424749e-02,  2.85305525e-03, -1.21543240e-02, ...,
          2.15355158e-02, -6.34655217e-03,  1.02934204e-02],
        ...,
        [ 3.62815685e-04,  3.94196855e-03,  8.03928822e-03, ...,
         -2.90077459e-03, -5.93607372e-04,  3.15086567e-03],
        [-1.50619149e-02, -2.51054447e-02, -2.12774873e-02, ...,
         -3.54218693e-03,  2.81670527e-03, -1.14895822e-02],
        [ 1.07423374e-02, -7.61732575e-04,  3.77989337e-02, ...,
          9.42581892e-03,  1.79382302e-02, -4.48567374e-03]]],      dtype=float32)
                },
                'query': {
                    'b': Array([[ 0.01000108,  0.01639372,  0.00138619, ...,  0.00279542,
         0.01172931, -0.01006246],
       [-0.00861747, -0.00524879,  0.01512013, ..., -0.01792894,
         0.0165606 ,  0.01120927],
       [ 0.01261996,  0.0112497 , -0.00451962, ..., -0.01455342,
        -0.01891823, -0.01427975],
       ...,
       [-0.01672397, -0.00771439,  0.00280647, ..., -0.0115596 ,
         0.00929647, -0.00382957],
       [ 0.0151234 ,  0.01519945,  0.03095485, ..., -0.01198111,
         0.00296444,  0.01389938],
       [-0.01285456, -0.00179988,  0.00022462, ..., -0.00894534,
        -0.01765853, -0.00730262]], dtype=float32),
                    'dora_m': Array([[[0.8578489 , 0.8517605 , 0.7332851 , ..., 0.6392991 ,
         0.9094501 , 0.81387234],
        [0.6589576 , 0.7280343 , 0.8099894 , ..., 0.7128408 ,
         0.7053289 , 0.7086578 ],
        [0.59198934, 0.646335  , 0.68926847, ..., 0.5895039 ,
         0.7240331 , 0.7621634 ],
        ...,
        [0.6576682 , 0.7099851 , 0.75339395, ..., 0.78092945,
         0.7273541 , 0.6942419 ],
        [0.67294085, 0.6623407 , 0.8342539 , ..., 0.7712699 ,
         0.73146445, 0.80962986],
        [0.7900105 , 0.7070912 , 0.8686833 , ..., 0.76999706,
         0.8380717 , 0.60558575]]], dtype=float32),
                    'lora_a': Array([[-0.1084516 ],
       [ 0.2945396 ],
       [-0.2603807 ],
       ...,
       [-0.7982564 ],
       [ 0.73765296],
       [-0.08399874]], dtype=float32),
                    'lora_b': Array([[-2.0432302e-05],
       [ 1.7916771e-05],
       [ 1.9953522e-05],
       ...,
       [-8.8196994e-06],
       [ 1.7037559e-05],
       [-1.4552521e-05]], dtype=float32),
                    'w': Array([[[-0.02677594,  0.00599007, -0.00895208, ..., -0.00217276,
          0.0255127 , -0.00099586],
        [ 0.00824958,  0.00178674, -0.01930125, ..., -0.04208021,
         -0.00241481, -0.04280047],
        [ 0.03461367,  0.02654499, -0.01111733, ..., -0.00716423,
          0.02020551, -0.05437608],
        ...,
        [-0.02426664, -0.01644287,  0.00571987, ..., -0.04034445,
          0.0047169 , -0.01945657],
        [ 0.00492201, -0.03032579,  0.01060695, ..., -0.03597231,
          0.04792401, -0.00643607],
        [ 0.0501137 ,  0.0061601 , -0.03792244, ..., -0.03496204,
         -0.02160517,  0.0073292 ]],

...,

[[-0.0056584 , -0.03200392, -0.01644075, ..., -0.03680464,
         -0.00148697,  0.02020196],
        [ 0.00363839,  0.03369744, -0.02185607, ...,  0.02236145,
         -0.04222722,  0.00038236],
        [ 0.05848982, -0.01653695, -0.0145413 , ...,  0.0178215 ,
          0.01025046, -0.04149369],
        ...,
        [ 0.01977103, -0.00522108, -0.01307487, ...,  0.03509489,
         -0.00274744, -0.00983507],
        [-0.0104529 , -0.01347402, -0.0346458 , ..., -0.02915582,
          0.03944222,  0.00634644],
        [ 0.01344375,  0.00525091, -0.03533768, ...,  0.00530439,
         -0.01809248,  0.00764443]]], dtype=float32)
                },
                'value': {
                    'b': Array([[ 1.6401194e-03,  5.3922208e-03, -5.0728004e-03, ...,
        -1.8111293e-03,  3.5093396e-04, -2.6095917e-03],
       [-1.7891753e-03, -7.4543618e-04, -1.1999641e-05, ...,
        -2.5042202e-03, -4.3558124e-03,  2.0704704e-04],
       [-1.1988505e-03, -1.7449020e-03,  5.3674653e-03, ...,
        -8.3908770e-04, -2.0188882e-03,  1.9707580e-03],
       ...,
       [-3.9382582e-03, -1.8472752e-03,  2.7255272e-03, ...,
         1.2182797e-03, -1.7120343e-05, -9.9033071e-04],
       [-2.3493757e-03, -6.2046667e-05,  2.9998010e-03, ...,
         2.6661516e-03, -3.9518168e-03, -1.3634117e-03],
       [ 2.6297674e-03,  2.2432012e-03,  9.0688950e-04, ...,
         2.9568389e-04, -4.6239821e-03, -2.7873907e-03]], dtype=float32),
                    'dora_m': Array([[[0.42094138, 0.34829566, 0.36897624, ..., 0.496673  ,
         0.5427811 , 0.42728117],
        [0.37377912, 0.43494827, 0.5292769 , ..., 0.48108974,
         0.57844836, 0.43604487],
        [0.34942374, 0.4536094 , 0.41075468, ..., 0.4690877 ,
         0.34185022, 0.416717  ],
        ...,
        [0.48667604, 0.4711419 , 0.43518695, ..., 0.3586769 ,
         0.33555034, 0.35114884],
        [0.49104893, 0.49999928, 0.38100234, ..., 0.4223346 ,
         0.34673733, 0.50167996],
        [0.43232167, 0.47632906, 0.41574886, ..., 0.5221746 ,
         0.50475556, 0.47249037]]], dtype=float32),
                    'lora_a': Array([[-0.10845059],
       [ 0.29454547],
       [-0.2603895 ],
       ...,
       [-0.7982609 ],
       [ 0.7376559 ],
       [-0.08399852]], dtype=float32),
                    'lora_b': Array([[ 1.9149587e-05],
       [-2.0572934e-05],
       [ 2.0977010e-05],
       ...,
       [-2.1951562e-05],
       [-2.0475765e-05],
       [ 1.3481302e-05]], dtype=float32),
                    'w': Array([[[-0.00406329,  0.00125176, -0.00566959, ...,  0.02132011,
          0.00988541,  0.03411871],
        [-0.00466405,  0.01822393,  0.03936379, ..., -0.01542882,
          0.02916765, -0.00973647],
        [ 0.00336628,  0.01916592,  0.01381161, ...,  0.0287559 ,
          0.00210257, -0.00523479],
        ...,
        [-0.00771339, -0.00339798, -0.01955114, ..., -0.00185548,
          0.00454612,  0.00495689],
        [-0.00247559,  0.0023475 , -0.00615943, ..., -0.01418786,
          0.00403497, -0.01032023],
        [ 0.01253546,  0.01948996,  0.00612835, ...,  0.03005783,
         -0.0113332 ,  0.00066234]],

...,

[[-0.0159568 ,  0.00502084, -0.02019401, ...,  0.00596474,
          0.0236849 , -0.02418589],
        [ 0.02489095,  0.02883545, -0.00887448, ...,  0.00603098,
          0.0235995 , -0.01852634],
        [ 0.01750358,  0.00283267,  0.01605931, ...,  0.00164372,
          0.01845782, -0.00467066],
        ...,
        [ 0.01845044,  0.00277444, -0.00401578, ...,  0.01248607,
         -0.00714417, -0.01576691],
        [ 0.00713076,  0.02249625,  0.00833604, ...,  0.02588477,
         -0.00887445,  0.0239369 ],
        [ 0.0180273 ,  0.00425099, -0.00061115, ...,  0.01271264,
          0.0059908 , -0.00427213]]], dtype=float32)
                }
            }
        },
        'x_layers_18': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': Array([ 0.00051136, -0.01148895,  0.00169411, ..., -0.00297784,
       -0.00223244, -0.0024133 ], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[0.99486804, 1.3522987 , 1.5606372 , ..., 1.354201  , 1.1416168 ,
        1.0600983 ]], dtype=float32),
                        'lora_a': Array([[-0.1084486 ],
       [ 0.2945531 ],
       [-0.26034948],
       ...,
       [-0.79827857],
       [ 0.7376457 ],
       [-0.08400054]], dtype=float32),
                        'lora_b': Array([[-9.7370284e-06],
       [ 8.8205006e-06],
       [-3.6279364e-06],
       ...,
       [ 0.0000000e+00],
       [-1.8762075e-05],
       [ 2.2096465e-05]], dtype=float32),
                        'w': Array([[ 0.00378705, -0.04327458,  0.07254019, ..., -0.04718518,
         0.03434655,  0.0322656 ],
       [ 0.02538404,  0.02334474, -0.03720424, ...,  0.01903304,
        -0.01485185,  0.04608155],
       [ 0.02892762, -0.04424008, -0.02908011, ..., -0.07568386,
        -0.02110019, -0.01552906],
       ...,
       [ 0.03719405, -0.02541165,  0.05467693, ..., -0.08789484,
        -0.02933055, -0.00520746],
       [-0.03056414, -0.00606433,  0.08469906, ...,  0.09002001,
         0.02983194,  0.02113312],
       [ 0.0071219 , -0.00418697,  0.07024559, ..., -0.06116322,
        -0.05130556, -0.04037147]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': Array([-0.00182306,  0.00342986,  0.00286894, ..., -0.005461  ,
       -0.00311243,  0.00018258], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.4907928, 1.5091332, 1.397797 , ..., 1.5834683, 1.3851281,
        1.4310509]], dtype=float32),
                        'lora_a': Array([[-0.10846459],
       [ 0.29453444],
       [-0.26037487],
       ...,
       [-0.7982576 ],
       [ 0.7376362 ],
       [-0.08404052]], dtype=float32),
                        'lora_b': Array([[ 1.3361388e-05],
       [ 6.9736229e-06],
       [ 2.0945698e-05],
       ...,
       [ 4.7025364e-06],
       [ 1.9562915e-05],
       [-1.8367833e-05]], dtype=float32),
                        'w': Array([[ 0.01468336,  0.02432303, -0.00566332, ..., -0.03545314,
         0.00487696,  0.03696332],
       [-0.00037203,  0.0256752 ,  0.02414542, ...,  0.06217912,
         0.00418166, -0.07536559],
       [ 0.00479466, -0.02747266, -0.04529685, ..., -0.03484176,
         0.05739212,  0.09449966],
       ...,
       [-0.06952304,  0.05575815,  0.00547075, ..., -0.01663679,
         0.10182662, -0.08988286],
       [ 0.03011166,  0.02201597, -0.0587035 , ..., -0.02506955,
         0.03673257,  0.02384413],
       [-0.03572247,  0.02936964, -0.0085162 , ...,  0.03586204,
        -0.00344068,  0.0381567 ]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': Array([-1.6253753e-04,  2.0698961e-05,  4.9653015e-04, ...,
        1.8806861e-03, -7.9710046e-03, -6.5031382e-03], dtype=float32),
                    'scale': Array([0.03690767, 0.01909823, 0.0559881 , ..., 0.04496968, 0.05597709,
       0.0311148 ], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': Array([1.1104548, 1.1232488, 1.1060338, ..., 1.0945585, 1.1172719,
       1.0616626], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': Array([[ 4.62999305e-04, -9.51305556e-04, -7.77834328e-04, ...,
        -1.01630467e-04, -1.42782566e-03, -1.38255733e-03],
       [-3.89249367e-03,  1.00802258e-03, -3.89283267e-03, ...,
        -1.02095969e-03,  4.24970029e-04,  6.21549669e-04],
       [ 1.71094539e-03, -2.22824907e-04, -2.29796919e-04, ...,
         6.06717891e-04, -1.62431935e-03,  8.83733795e-04],
       ...,
       [ 1.87748508e-03,  4.73401946e-04, -5.41095797e-04, ...,
        -1.95898069e-03, -4.02874604e-04, -1.82685454e-03],
       [ 2.49577849e-03, -6.80000638e-04,  4.40471398e-04, ...,
         5.82880108e-04,  3.10799922e-03,  4.94156266e-04],
       [-1.02130929e-03,  5.47624040e-05, -5.23315066e-05, ...,
        -1.04511355e-03, -3.17567843e-04,  1.13948445e-04]],      dtype=float32),
                    'dora_m': Array([[[0.5389666 , 0.6147816 , 0.6080973 , ..., 0.61912006,
         0.6289355 , 0.5739352 ],
        [0.39015514, 0.44474587, 0.47647834, ..., 0.51906157,
         0.3894832 , 0.4837889 ],
        [0.57488567, 0.6261492 , 0.5152367 , ..., 0.56964207,
         0.43689805, 0.6440447 ],
        ...,
        [0.5284905 , 0.4620954 , 0.5646536 , ..., 0.5259102 ,
         0.70120287, 0.5618357 ],
        [0.5828247 , 0.5550093 , 0.6565689 , ..., 0.79493016,
         0.49571994, 0.5169428 ],
        [0.5333974 , 0.6261711 , 0.59194916, ..., 0.60567844,
         0.5645008 , 0.6600115 ]]], dtype=float32),
                    'lora_a': Array([[-0.1084687 ],
       [ 0.29451683],
       [-0.26037437],
       ...,
       [-0.798245  ],
       [ 0.7376523 ],
       [-0.0840354 ]], dtype=float32),
                    'lora_b': Array([[ 1.4560178e-05],
       [-2.0973941e-05],
       [-1.3534562e-06],
       ...,
       [ 2.1359529e-05],
       [-1.5850253e-05],
       [-1.3696293e-05]], dtype=float32),
                    'w': Array([[[-1.25487559e-02,  5.27682714e-03, -8.67328432e-04, ...,
         -1.05484826e-02,  6.49900082e-03, -1.18660368e-02],
        [-6.30257931e-03,  5.88576403e-03, -2.17002351e-03, ...,
          2.14710347e-02,  1.67809043e-03, -1.66181996e-02],
        [ 4.71557165e-03, -3.11150122e-03, -1.95540395e-02, ...,
          4.38414188e-03, -8.43202753e-04,  3.24525163e-02],
        ...,
        [-7.45568192e-04,  1.49174854e-02, -1.20049333e-02, ...,
          2.66597308e-02, -5.00853918e-03,  1.92217473e-02],
        [-4.22099652e-03, -1.88382305e-02,  2.46013161e-02, ...,
          1.17878467e-02, -5.46755921e-03,  1.35662928e-02],
        [-1.58921070e-02,  1.14033790e-02,  9.26128123e-03, ...,
         -1.08587695e-02,  1.97254512e-02, -3.81363183e-02]],

...,

[[ 1.09337596e-02, -2.10437980e-02, -1.05800685e-02, ...,
         -6.56963792e-03,  1.45507127e-03,  8.91961996e-03],
        [-3.75897298e-03, -6.11150730e-03,  2.85932445e-03, ...,
          1.28849866e-02,  3.19971680e-03, -5.58700413e-04],
        [ 5.85980422e-04, -4.57300013e-03, -1.03927888e-02, ...,
         -4.13403697e-02, -1.30845634e-02,  1.71099808e-02],
        ...,
        [-2.68593943e-03,  3.69987800e-04,  1.51738245e-03, ...,
          4.83785942e-03,  1.66357215e-02, -5.48439845e-03],
        [ 1.20854368e-02,  5.05305175e-03,  2.46214960e-02, ...,
         -3.43978219e-02,  7.53583247e-03,  3.36864591e-02],
        [-1.13962796e-02, -1.49790356e-02, -1.38309961e-02, ...,
          6.29504793e-04, -1.78377591e-02, -1.49134332e-02]]],      dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': Array([0.14654781, 0.22278094, 0.2743674 , 0.20907548, 0.24956009,
       0.18405011, 0.21791767, 0.22734475, 0.2668076 , 0.20462586,
       0.19914745, 0.28613123, 0.23778196, 0.27146855, 0.23280822,
       0.20081756, 0.19908659, 0.18805924, 0.23253727, 0.18743424,
       0.1948078 , 0.17780925, 0.19240059, 0.20284435, 0.23953158,
       0.2414018 , 0.22483683, 0.27306545, 0.1580696 , 0.23140562,
       0.2652837 , 0.25217816, 0.25966236, 0.21986026, 0.1649964 ,
       0.2737241 , 0.21450056, 0.2052232 , 0.2948621 , 0.2333531 ,
       0.21151926, 0.17604513, 0.20712714, 0.27924615, 0.23490104,
       0.25804925, 0.30132753, 0.25481915, 0.26180637, 0.28384343,
       0.29114008, 0.2278892 , 0.13483137, 0.28901073, 0.22650169,
       0.19220173, 0.20297801, 0.24761178, 0.22846888, 0.22040966,
       0.27271867, 0.31886327, 0.24663903, 0.20238942, 0.21435668,
       0.18714531, 0.23238699, 0.26992947, 0.20557305, 0.18135974,
       0.1717081 , 0.13401309, 0.20160556, 0.24493887, 0.15000689,
       0.24136183, 0.25651586, 0.2202598 , 0.20814568, 0.27010095],      dtype=float32)
                },
                'post': {
                    'b': Array([-0.00229272,  0.00798599,  0.00748841, ..., -0.00560183,
       -0.00854836, -0.00232937], dtype=float32),
                    'dora_m': Array([[[0.6815765 , 0.6290562 , 0.67502856, ..., 0.60667413,
         0.5988507 , 0.61164314],
        [0.62812734, 0.6038167 , 0.37679195, ..., 0.5648584 ,
         0.63888824, 0.32882467],
        [0.39299768, 0.5104977 , 0.5433063 , ..., 0.33232254,
         0.3348628 , 0.5404346 ],
        ...,
        [0.59671414, 0.45427302, 0.5881536 , ..., 0.6005097 ,
         0.65705377, 0.43177262],
        [0.3347007 , 0.5359568 , 0.6671518 , ..., 0.5371677 ,
         0.6742836 , 0.53683996],
        [0.50998163, 0.6339325 , 0.66276485, ..., 0.55114305,
         0.6098107 , 0.59093285]]], dtype=float32),
                    'lora_a': Array([[-0.10845607],
       [ 0.29454195],
       [-0.26035208],
       ...,
       [-0.79823977],
       [ 0.7376492 ],
       [-0.08400579]], dtype=float32),
                    'lora_b': Array([[-1.4999511e-05],
       [-9.3675244e-06],
       [ 7.4600266e-06],
       ...,
       [ 1.7859269e-05],
       [-1.7138622e-05],
       [-1.6737913e-05]], dtype=float32),
                    'w': Array([[[-0.00671915,  0.00587748, -0.01147179, ...,  0.01086278,
          0.00020015, -0.0060481 ],
        [ 0.00825962, -0.00934129, -0.00360557, ...,  0.0052317 ,
          0.01242181, -0.00182717],
        [-0.00623296,  0.00317535,  0.00204991, ...,  0.00817461,
          0.00050625,  0.01241348],
        ...,
        [-0.01216628,  0.0058833 ,  0.00036413, ...,  0.00734476,
         -0.00440068, -0.00117811],
        [-0.00370096,  0.00782828,  0.00343848, ..., -0.00323366,
         -0.01976196,  0.01709105],
        [-0.01143755, -0.00773526, -0.03307534, ...,  0.0074515 ,
         -0.01465548, -0.00724028]],

...,

[[ 0.00178173, -0.0117442 , -0.0419663 , ...,  0.00717429,
         -0.0201303 ,  0.02430891],
        [ 0.00307834, -0.02651902, -0.01245268, ..., -0.00345665,
         -0.01771973, -0.00738682],
        [-0.02064985, -0.00220527, -0.0091716 , ..., -0.00358126,
         -0.01540149, -0.0127347 ],
        ...,
        [-0.01497488,  0.00410123,  0.00180891, ...,  0.00390501,
          0.01068672, -0.00476688],
        [ 0.01522674,  0.00994318,  0.02885547, ...,  0.00942743,
          0.03936775,  0.02158558],
        [ 0.01009504,  0.00526136, -0.0112782 , ..., -0.00397819,
         -0.01786978,  0.01006007]]], dtype=float32)
                },
                'query': {
                    'b': Array([[-0.01693531,  0.01281131,  0.00499396, ...,  0.00779316,
        -0.00025738,  0.01801906],
       [-0.02124937,  0.00200343, -0.00883132, ...,  0.00201107,
         0.00554235,  0.01515533],
       [-0.00305617,  0.004093  , -0.0067767 , ...,  0.00522619,
         0.01738874, -0.00898235],
       ...,
       [-0.0055667 , -0.0178722 ,  0.01464478, ...,  0.00780344,
         0.00424719,  0.00944584],
       [ 0.017694  , -0.02612935,  0.0004033 , ..., -0.0020163 ,
         0.01824481,  0.01316419],
       [ 0.02740095, -0.00695968, -0.00573279, ...,  0.00210098,
         0.01201493,  0.00295716]], dtype=float32),
                    'dora_m': Array([[[0.937613  , 0.7032686 , 0.76778865, ..., 0.65307647,
         0.92838705, 0.9566522 ],
        [0.58410484, 0.60784346, 0.63368845, ..., 0.7446482 ,
         0.58931637, 0.6759172 ],
        [0.7987242 , 0.8131232 , 0.764702  , ..., 0.7553525 ,
         0.80644286, 0.79187447],
        ...,
        [0.9751542 , 0.81482065, 1.0045047 , ..., 0.6380343 ,
         0.8012705 , 0.8350481 ],
        [0.8417992 , 1.0036376 , 0.80287683, ..., 0.72015494,
         0.82166046, 0.93767875],
        [0.9835473 , 0.9208775 , 0.6171657 , ..., 0.6475353 ,
         0.84889156, 0.7494903 ]]], dtype=float32),
                    'lora_a': Array([[-0.10844965],
       [ 0.29455096],
       [-0.26035726],
       ...,
       [-0.79827785],
       [ 0.7376226 ],
       [-0.08400004]], dtype=float32),
                    'lora_b': Array([[ 1.3426383e-05],
       [-1.9420419e-05],
       [-1.8559813e-05],
       ...,
       [-1.5290991e-05],
       [-1.6041215e-06],
       [ 3.2139474e-06]], dtype=float32),
                    'w': Array([[[-0.04122668,  0.03792162,  0.01956038, ...,  0.00083074,
         -0.02338985,  0.00243163],
        [-0.00712831,  0.01235137, -0.02554855, ..., -0.01994032,
          0.00118904, -0.00247901],
        [-0.01280304, -0.00637479,  0.02810878, ...,  0.00658902,
          0.03580311, -0.03428695],
        ...,
        [-0.02720779, -0.01454407,  0.02582767, ..., -0.02545251,
         -0.01254367, -0.05030369],
        [-0.0039447 , -0.02432956, -0.00034708, ..., -0.01586003,
          0.01817022,  0.03687435],
        [-0.003418  ,  0.02158768, -0.00758175, ...,  0.00150226,
         -0.04043954, -0.01866934]],

...,

[[ 0.01294661, -0.01451382, -0.01001048, ...,  0.03827586,
          0.02376596,  0.00691161],
        [ 0.01033855, -0.01908808, -0.00044221, ..., -0.01828135,
          0.0102315 , -0.0078474 ],
        [ 0.02605218,  0.05498118,  0.04802718, ...,  0.03354565,
         -0.02080743, -0.01015075],
        ...,
        [-0.02980904,  0.01638061, -0.01147437, ..., -0.0347767 ,
          0.0278121 ,  0.00401018],
        [ 0.0189371 , -0.00872775, -0.03812696, ..., -0.02774808,
          0.00812404,  0.01737259],
        [ 0.02776276,  0.01406731, -0.01994073, ...,  0.01034588,
         -0.02899795, -0.01627316]]], dtype=float32)
                },
                'value': {
                    'b': Array([[-0.00188822, -0.00093386,  0.00192111, ...,  0.00245047,
         0.00135479,  0.001175  ],
       [ 0.0006789 ,  0.00084782, -0.00147296, ..., -0.00530237,
         0.00315595, -0.00136973],
       [-0.00260676,  0.00303111, -0.00157738, ..., -0.00982555,
        -0.01084454,  0.00202332],
       ...,
       [-0.0045933 , -0.00203996,  0.00438417, ..., -0.00307284,
        -0.00089111,  0.00201392],
       [-0.0005301 , -0.00109679, -0.0009102 , ..., -0.00083201,
         0.00086958, -0.0019664 ],
       [-0.00357503,  0.00066696,  0.00028513, ...,  0.00167007,
        -0.0029751 , -0.0002654 ]], dtype=float32),
                    'dora_m': Array([[[0.6249877 , 0.6649066 , 0.6357403 , ..., 0.61342317,
         0.55516106, 0.55826515],
        [0.60650253, 0.60699993, 0.3571156 , ..., 0.53671056,
         0.5985569 , 0.3332211 ],
        [0.4728521 , 0.5679459 , 0.5850894 , ..., 0.39518243,
         0.38843334, 0.52792054],
        ...,
        [0.5727818 , 0.51226574, 0.5560117 , ..., 0.57297117,
         0.601528  , 0.50281817],
        [0.33484012, 0.5076307 , 0.5919125 , ..., 0.5402243 ,
         0.56933624, 0.54217213],
        [0.5549892 , 0.6323444 , 0.6486045 , ..., 0.5858656 ,
         0.59277093, 0.5946604 ]]], dtype=float32),
                    'lora_a': Array([[-0.10847948],
       [ 0.29455158],
       [-0.26035136],
       ...,
       [-0.79827917],
       [ 0.7376491 ],
       [-0.08399864]], dtype=float32),
                    'lora_b': Array([[ 2.0305315e-05],
       [ 8.8189508e-06],
       [-1.1246420e-05],
       ...,
       [-2.0945783e-05],
       [ 1.4021880e-05],
       [ 5.7390059e-07]], dtype=float32),
                    'w': Array([[[ 7.46658957e-03, -8.41677841e-03,  3.31376195e-02, ...,
         -2.65153358e-03, -3.16191697e-04,  1.41285108e-02],
        [-3.97422947e-02, -1.77706983e-02, -3.90480249e-03, ...,
          1.01006670e-04, -1.42450503e-03, -1.40604619e-02],
        [-1.60975344e-02,  6.63828477e-03,  3.24483099e-03, ...,
         -2.85821017e-02,  1.02422163e-02, -2.28672903e-02],
        ...,
        [-1.64939910e-02,  1.04231611e-02, -3.46118659e-02, ...,
          8.36394448e-03, -2.87051387e-02, -7.90313631e-03],
        [-3.05770291e-03,  4.16853727e-04,  2.16026939e-02, ...,
          1.48915090e-02, -1.02293175e-02,  4.35859896e-03],
        [-6.22540386e-03, -1.09573379e-02, -4.03586263e-03, ...,
          2.01954097e-02,  8.48009065e-03, -1.28198322e-02]],

...,

[[ 9.88735631e-03,  1.84934516e-03,  1.97978858e-02, ...,
         -4.30854387e-04, -7.47772772e-03, -2.07434222e-02],
        [ 1.15765315e-02,  4.83674072e-02,  6.84570847e-03, ...,
          9.09647997e-03, -1.43847680e-02,  2.62443088e-02],
        [ 5.19147981e-03,  1.08308000e-02, -1.10734832e-02, ...,
          8.27481411e-03,  1.91579498e-02,  3.50668430e-02],
        ...,
        [ 3.39808539e-02,  5.29606361e-03, -6.18779194e-03, ...,
         -2.93455273e-02, -1.68223586e-02,  1.95688456e-02],
        [-2.16533802e-02, -1.91696882e-02,  4.10865666e-03, ...,
         -1.37714809e-02, -1.17597580e-02,  1.65862951e-03],
        [ 2.24986691e-02, -3.52053496e-04, -1.82409468e-03, ...,
          1.69674065e-02,  4.27982099e-02,  6.34616241e-03]]],      dtype=float32)
                }
            }
        },
        'x_layers_19': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': Array([-0.00279087,  0.00553406, -0.00618837, ..., -0.00082947,
       -0.00121525,  0.00695249], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.0707939, 1.0929668, 1.018043 , ..., 1.4054546, 1.2883576,
        1.0494723]], dtype=float32),
                        'lora_a': Array([[-0.10844899],
       [ 0.29455304],
       [-0.26035407],
       ...,
       [-0.79827327],
       [ 0.73765063],
       [-0.08400065]], dtype=float32),
                        'lora_b': Array([[-6.44688589e-06],
       [ 7.66394714e-06],
       [-1.25184015e-05],
       ...,
       [-1.38350142e-05],
       [-2.02348747e-06],
       [-1.08328359e-05]], dtype=float32),
                        'w': Array([[-0.03057937, -0.03449765, -0.00030052, ..., -0.04533616,
        -0.01315903,  0.00346747],
       [-0.0189806 , -0.0139573 , -0.0167984 , ..., -0.01144543,
        -0.03284094, -0.05118792],
       [-0.03204586, -0.01559104, -0.03456049, ..., -0.05110538,
         0.01989403, -0.0404238 ],
       ...,
       [ 0.01285772, -0.03061882,  0.02235553, ..., -0.02296763,
        -0.01331782,  0.03847779],
       [-0.03716642, -0.03083301, -0.02233933, ...,  0.01910252,
        -0.01984283,  0.01339958],
       [-0.01486247, -0.03447136,  0.02612187, ..., -0.00439616,
         0.02521509,  0.00436651]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': Array([-0.00071812,  0.00626113,  0.00534002, ..., -0.00520115,
       -0.00256186,  0.00024727], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.2743475, 1.6168783, 1.8151804, ..., 1.537862 , 1.3753194,
        1.1602237]], dtype=float32),
                        'lora_a': Array([[-0.10845133],
       [ 0.29453894],
       [-0.26037115],
       ...,
       [-0.79825467],
       [ 0.7376533 ],
       [-0.08400157]], dtype=float32),
                        'lora_b': Array([[ 2.2578606e-06],
       [ 1.0978436e-05],
       [-3.0405845e-06],
       ...,
       [ 1.2955283e-05],
       [-1.3890548e-05],
       [-9.9871640e-06]], dtype=float32),
                        'w': Array([[-0.00638571, -0.03181123,  0.01637862, ..., -0.02804339,
         0.01380809, -0.01930169],
       [ 0.03387563,  0.03299653, -0.0156489 , ...,  0.04582642,
         0.00388483,  0.0091139 ],
       [-0.02489925, -0.0313705 , -0.00089129, ...,  0.03598975,
        -0.01924329,  0.00141226],
       ...,
       [ 0.0071869 , -0.01164675, -0.12252312, ..., -0.04268306,
         0.03268336,  0.02160267],
       [-0.02873901,  0.03516197, -0.00716065, ..., -0.0070508 ,
        -0.02109207,  0.01088641],
       [ 0.03129392,  0.04549291, -0.00161096, ..., -0.02231779,
        -0.00662258, -0.04896937]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': Array([-0.00068695, -0.000179  , -0.00298053, ..., -0.00083815,
       -0.00157204,  0.00269903], dtype=float32),
                    'scale': Array([0.01983866, 0.03065088, 0.06953451, ..., 0.05586866, 0.07119216,
       0.00991571], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': Array([1.12914  , 1.1227869, 1.1314727, ..., 1.0988649, 1.0992268,
       1.0550259], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': Array([[ 5.6358091e-05,  8.3575916e-04, -5.3359644e-04, ...,
         4.5242821e-04,  5.4248731e-04,  4.3794292e-04],
       [ 5.3405447e-04, -3.8205500e-04,  1.9125095e-04, ...,
        -1.8305663e-04,  2.3835654e-04, -4.2093513e-04],
       [-1.5286071e-03,  5.1494699e-04, -7.5894699e-04, ...,
        -2.0895094e-04, -1.0857989e-04,  3.8178050e-04],
       ...,
       [-7.7942845e-05, -1.5454808e-03,  1.1539065e-03, ...,
        -1.2183716e-03, -9.4440789e-04, -8.2142715e-04],
       [ 1.9358263e-04, -2.3736360e-03,  1.2290434e-04, ...,
        -7.5245055e-04, -6.6944674e-05, -3.7660662e-03],
       [-1.1276773e-03, -8.6710806e-04,  8.2271406e-04, ...,
        -4.8099627e-04, -4.1263181e-04, -4.7033969e-05]], dtype=float32),
                    'dora_m': Array([[[0.4345917 , 0.47389168, 0.56106913, ..., 0.49411893,
         0.47209027, 0.4206815 ],
        [0.7910191 , 0.67552555, 0.65892905, ..., 0.65421474,
         0.7654362 , 0.632919  ],
        [0.49812922, 0.6458979 , 0.5418668 , ..., 0.48996085,
         0.63637394, 0.59923875],
        ...,
        [0.5381918 , 0.5101199 , 0.571651  , ..., 0.5488743 ,
         0.6102227 , 0.5040438 ],
        [1.1352146 , 0.7551614 , 0.858993  , ..., 0.7581285 ,
         0.842849  , 0.62257266],
        [0.5042739 , 0.35342613, 0.29661652, ..., 0.29237008,
         0.44864026, 0.32330582]]], dtype=float32),
                    'lora_a': Array([[-0.10847009],
       [ 0.2945148 ],
       [-0.260355  ],
       ...,
       [-0.7982452 ],
       [ 0.7376545 ],
       [-0.08403485]], dtype=float32),
                    'lora_b': Array([[-1.15468447e-05],
       [-1.81439027e-05],
       [-1.63381646e-05],
       ...,
       [-1.12505795e-05],
       [ 1.31390871e-05],
       [ 1.59223673e-05]], dtype=float32),
                    'w': Array([[[ 1.66858211e-02, -2.23971736e-02,  1.98825193e-03, ...,
         -1.66950971e-02,  2.84126699e-02,  6.86215423e-03],
        [-3.61021198e-02, -5.99008403e-04, -2.17522564e-03, ...,
         -5.36645204e-03, -1.71211548e-02,  7.97853246e-03],
        [-6.02039229e-03, -7.07927393e-03,  1.50340544e-02, ...,
         -1.07782977e-02,  6.55275024e-03,  7.70088658e-03],
        ...,
        [ 3.66256194e-04, -2.85069866e-04, -9.70526133e-03, ...,
          1.36355665e-02, -6.33951742e-03,  7.70825311e-04],
        [-1.41819892e-02,  1.63590889e-02,  2.55743060e-02, ...,
          2.41388604e-02, -5.91733120e-02,  1.40847396e-02],
        [-8.19051638e-03,  7.65746576e-04,  1.45718094e-03, ...,
          1.06700966e-02,  7.19248224e-03,  7.06929830e-04]],

...,

[[-5.90128312e-03,  7.19144708e-03, -1.62249047e-03, ...,
          7.17088347e-03,  2.48752953e-03, -1.46825088e-03],
        [-3.90324765e-03, -1.09524617e-03, -3.75927798e-03, ...,
         -2.10372284e-02, -3.45567288e-03, -2.17198627e-03],
        [-1.24420477e-02,  8.93982965e-03, -1.07633350e-02, ...,
          8.87843035e-03,  1.86779350e-02, -1.95613485e-02],
        ...,
        [ 5.58171421e-04,  4.86980774e-04, -2.23068683e-03, ...,
          5.12370095e-03,  1.37364306e-02, -3.50908330e-03],
        [-3.38176861e-02,  7.37074669e-03, -1.00086094e-03, ...,
         -5.08754030e-02, -4.78045829e-02, -2.40573566e-02],
        [ 2.48386350e-04,  5.62367728e-03,  9.40514356e-03, ...,
         -4.55174455e-03, -4.08558873e-03,  5.38862892e-04]]],      dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': Array([0.14652345, 0.16987266, 0.09893259, 0.08280052, 0.13477588,
       0.12619452, 0.1363653 , 0.07585543, 0.20202784, 0.08150236,
       0.18895143, 0.20548019, 0.10704212, 0.20818919, 0.17153841,
       0.14148037, 0.1626222 , 0.23672125, 0.1762036 , 0.1408799 ,
       0.2247247 , 0.20127554, 0.11450737, 0.23013239, 0.16911133,
       0.174619  , 0.25150812, 0.08649732, 0.12320526, 0.18035066,
       0.26151782, 0.19074674, 0.18112373, 0.07410798, 0.1610882 ,
       0.18994205, 0.12926589, 0.18631133, 0.1497615 , 0.22104244,
       0.10986016, 0.1717358 , 0.12376273, 0.1916074 , 0.14726362,
       0.21275796, 0.21343718, 0.20106877, 0.13625608, 0.12418237,
       0.12081207, 0.13094804, 0.23173405, 0.17589185, 0.14757948,
       0.16841263, 0.23804408, 0.18274456, 0.18690562, 0.21431428,
       0.22474097, 0.10995922, 0.15971506, 0.13611062, 0.16957259,
       0.17104879, 0.1870232 , 0.17258014, 0.13192719, 0.19396555,
       0.13188697, 0.21852025, 0.16542722, 0.15202987, 0.10554535,
       0.10272837, 0.14247337, 0.1691001 , 0.17632973, 0.11661879],      dtype=float32)
                },
                'post': {
                    'b': Array([-0.00091276,  0.00443273,  0.00454847, ..., -0.00962823,
        0.00059259, -0.0012725 ], dtype=float32),
                    'dora_m': Array([[[0.49136007, 0.5165123 , 0.40485543, ..., 0.57613146,
         0.73858166, 0.47206616],
        [0.5565406 , 0.5085424 , 0.52160895, ..., 0.43052712,
         0.583304  , 0.57699037],
        [0.5251394 , 0.7970782 , 0.6616093 , ..., 0.47278884,
         0.46289626, 0.6284039 ],
        ...,
        [0.6944826 , 0.80210924, 0.6286307 , ..., 0.5574507 ,
         0.88375074, 0.6577677 ],
        [0.5331752 , 0.59659606, 0.40244964, ..., 0.41858697,
         0.72679174, 0.5614958 ],
        [0.41619015, 0.37569445, 0.41099977, ..., 0.34426746,
         0.4616174 , 0.35712585]]], dtype=float32),
                    'lora_a': Array([[-0.1084714 ],
       [ 0.29454878],
       [-0.2603555 ],
       ...,
       [-0.7982368 ],
       [ 0.7376555 ],
       [-0.08400269]], dtype=float32),
                    'lora_b': Array([[-1.6540927e-05],
       [-1.5396210e-05],
       [-8.7920189e-06],
       ...,
       [ 1.7985043e-05],
       [-1.7888451e-05],
       [-1.8718492e-05]], dtype=float32),
                    'w': Array([[[-7.94719532e-03, -2.65852176e-03,  1.21650063e-02, ...,
          2.65658367e-04, -1.12542706e-02,  2.68825162e-02],
        [-3.55421146e-03, -1.01088062e-02,  1.50485346e-02, ...,
         -9.76439938e-03,  2.94282492e-02, -9.73171555e-03],
        [ 5.18939516e-04,  1.81097053e-02,  6.01419248e-04, ...,
          1.18027185e-03, -7.42555130e-03,  2.54618265e-02],
        ...,
        [-1.65492687e-02,  3.55440169e-03,  2.95281899e-03, ...,
         -2.62755007e-02, -1.22740353e-02, -1.86174165e-03],
        [-3.05727236e-02,  1.08236717e-02, -5.32779563e-03, ...,
         -5.65503817e-03,  2.51735039e-02, -5.72287897e-03],
        [-7.39056617e-03,  3.59133305e-03, -1.02642123e-02, ...,
         -5.22708427e-03,  1.66715204e-03, -1.24209216e-02]],

...,

[[-6.95926370e-03, -3.42211360e-03,  4.11564577e-03, ...,
          2.65010138e-04, -1.12122754e-02,  5.38297603e-03],
        [-4.65994468e-03, -1.20928045e-02,  1.39921932e-02, ...,
         -1.27126719e-03, -1.89517136e-03,  2.80775828e-03],
        [ 9.77591379e-04, -5.88644017e-03,  1.36731090e-02, ...,
          2.15155841e-03, -8.99187071e-05, -9.61266831e-03],
        ...,
        [ 1.08055631e-03, -3.01486365e-02, -7.61171943e-03, ...,
         -2.29782425e-03, -6.79991953e-03,  2.08318476e-02],
        [-1.46988519e-02, -1.81029423e-03,  3.89482989e-03, ...,
         -5.74233290e-03, -2.33056471e-02, -5.32351527e-03],
        [ 1.11967605e-02,  1.63350888e-02, -8.62249546e-03, ...,
          1.04464474e-03, -1.59137808e-02, -5.87823056e-03]]],      dtype=float32)
                },
                'query': {
                    'b': Array([[-2.3320248e-02, -1.1966302e-02, -2.0297846e-02, ...,
        -2.1983229e-02, -1.9539420e-02,  2.2182582e-02],
       [-5.8301748e-03,  8.1160599e-03, -6.9336863e-03, ...,
        -1.3168788e-02, -8.9469191e-05,  1.0755450e-02],
       [ 2.1333396e-02,  5.7952716e-03,  1.2760594e-03, ...,
         1.3478556e-02, -4.5719063e-03,  8.1177354e-03],
       ...,
       [-1.4207000e-03,  1.6074298e-02, -1.4178736e-02, ...,
         1.6082305e-02,  4.1668117e-03,  2.2911452e-02],
       [-7.9755876e-03, -1.5447275e-02,  1.0383511e-02, ...,
        -1.7911499e-02,  3.1178708e-03, -1.7663836e-02],
       [ 6.2362943e-03,  1.1733231e-02, -1.5588548e-02, ...,
         9.6454900e-03,  6.5174123e-04,  5.2759107e-03]], dtype=float32),
                    'dora_m': Array([[[0.6408635 , 0.8464077 , 0.81634575, ..., 0.95186543,
         0.88897747, 0.8690835 ],
        [0.7303971 , 0.77359223, 0.8089315 , ..., 1.0213987 ,
         0.7444608 , 0.72983533],
        [0.7676568 , 0.78313565, 0.69144064, ..., 0.65030825,
         0.8113023 , 0.80348915],
        ...,
        [0.7581582 , 0.7847238 , 0.92379373, ..., 0.7149467 ,
         0.90168387, 0.7636905 ],
        [0.8381095 , 1.1863424 , 1.1404012 , ..., 1.2748412 ,
         1.1165909 , 1.3532139 ],
        [0.54480565, 0.4638846 , 0.32995686, ..., 0.36605793,
         0.57822025, 0.38902223]]], dtype=float32),
                    'lora_a': Array([[-0.10844907],
       [ 0.2945514 ],
       [-0.2603507 ],
       ...,
       [-0.798276  ],
       [ 0.73764694],
       [-0.08400285]], dtype=float32),
                    'lora_b': Array([[ 1.3387214e-05],
       [ 1.0102771e-05],
       [ 1.6034661e-05],
       ...,
       [-1.7963906e-05],
       [-1.8424586e-05],
       [-1.0270741e-06]], dtype=float32),
                    'w': Array([[[ 0.01879046,  0.00627272,  0.01409843, ...,  0.02526905,
         -0.00265712,  0.05618662],
        [-0.02516003, -0.00326343,  0.04162131, ...,  0.00678667,
          0.01605123, -0.00443892],
        [ 0.02505029,  0.02248173, -0.01596485, ...,  0.00209537,
         -0.03365821,  0.05305671],
        ...,
        [ 0.01634249, -0.00203726, -0.03275561, ...,  0.00196546,
          0.0316908 ,  0.03987049],
        [-0.02480521, -0.0194451 ,  0.03565983, ...,  0.0883149 ,
          0.03553959, -0.0234833 ],
        [ 0.01779898,  0.00122205,  0.00608964, ..., -0.0020426 ,
         -0.01925553, -0.00076039]],

...,

[[-0.01105381,  0.00393943, -0.01289818, ..., -0.04812491,
          0.01336343, -0.01491917],
        [ 0.00134544,  0.0092295 , -0.0430508 , ..., -0.01045833,
          0.00656331,  0.00142347],
        [ 0.01488701, -0.03062654, -0.0072165 , ..., -0.02095346,
          0.01998034, -0.00554285],
        ...,
        [ 0.02381367, -0.01228064, -0.00435461, ..., -0.0181347 ,
          0.05034123, -0.00717285],
        [ 0.01958305, -0.04317761, -0.02946939, ...,  0.00058659,
          0.01312297,  0.0187073 ],
        [-0.02160233,  0.01526545,  0.01430138, ..., -0.00187082,
          0.02324795,  0.00513071]]], dtype=float32)
                },
                'value': {
                    'b': Array([[ 2.6284866e-03,  2.7871240e-04, -6.0292310e-03, ...,
        -7.3629955e-04,  1.2970131e-03, -3.3789459e-03],
       [-1.7902206e-03, -1.6401059e-03, -3.2759264e-05, ...,
        -6.9711115e-03, -1.6303200e-03,  2.0977818e-04],
       [-4.6065627e-05, -1.3298161e-03,  1.7794441e-03, ...,
         1.8877160e-03,  1.4194903e-03, -3.3245850e-03],
       ...,
       [-2.9255324e-03, -3.5976777e-03,  2.1950365e-03, ...,
         8.8367925e-04, -2.0187130e-05,  7.3943031e-04],
       [-2.8092548e-04,  1.5679412e-03,  1.5219685e-03, ...,
        -1.4931384e-03, -9.9890260e-04,  2.8214464e-03],
       [ 4.5016697e-03, -4.0114883e-04, -4.4532988e-04, ...,
         1.4430106e-04,  5.1092445e-03, -6.9501926e-04]], dtype=float32),
                    'dora_m': Array([[[0.5206604 , 0.46026507, 0.40729195, ..., 0.56922275,
         0.67639047, 0.45490164],
        [0.5372895 , 0.5613128 , 0.5535611 , ..., 0.48691857,
         0.64929426, 0.57059497],
        [0.50428915, 0.7703894 , 0.70110095, ..., 0.5205817 ,
         0.5700486 , 0.7947661 ],
        ...,
        [0.66312355, 0.79998255, 0.6333175 , ..., 0.60252655,
         0.8091594 , 0.65167755],
        [0.48364472, 0.55658174, 0.388526  , ..., 0.4247905 ,
         0.6164522 , 0.4955588 ],
        [0.44331923, 0.40080395, 0.44940594, ..., 0.37491202,
         0.45277846, 0.48657364]]], dtype=float32),
                    'lora_a': Array([[-0.10844879],
       [ 0.29455122],
       [-0.26035032],
       ...,
       [-0.7982778 ],
       [ 0.73765457],
       [-0.08400292]], dtype=float32),
                    'lora_b': Array([[ 2.9899550e-06],
       [-8.0304735e-06],
       [ 1.9225467e-05],
       ...,
       [ 2.1189722e-05],
       [-2.5661357e-06],
       [-2.2187553e-05]], dtype=float32),
                    'w': Array([[[ 9.36927367e-03,  2.24846452e-02,  2.45292974e-03, ...,
         -8.28975625e-03, -2.75090914e-02,  9.59235243e-03],
        [ 1.71270373e-03, -1.39735130e-04,  3.37577946e-02, ...,
         -4.37170640e-03, -3.14194057e-03, -2.15227436e-02],
        [-1.54672004e-03, -2.74255257e-02, -3.03714871e-02, ...,
          1.52025865e-02,  1.13139674e-03, -4.77866316e-03],
        ...,
        [ 3.19389775e-02, -5.01666870e-03, -1.28704822e-02, ...,
          5.77493198e-03,  8.06158991e-04, -5.89923875e-04],
        [ 3.38947400e-02, -1.96365695e-02, -1.01442384e-02, ...,
          5.47762588e-03, -2.87755374e-02, -2.06911899e-02],
        [ 2.48693340e-02,  5.00555011e-03,  7.18526216e-03, ...,
          2.34933128e-03,  9.19876806e-03,  3.31273419e-03]],

...,

[[ 6.23098109e-03,  1.28009240e-03,  1.90864019e-02, ...,
          3.05063510e-03, -1.20336991e-02, -7.22211367e-03],
        [ 2.42555980e-03,  1.72286611e-02, -5.33142500e-03, ...,
         -6.52101682e-03, -1.71236970e-04, -2.13873480e-03],
        [-1.23704579e-02,  3.44731025e-02, -8.13399721e-03, ...,
         -1.78678595e-02,  1.77270956e-02,  1.44108320e-02],
        ...,
        [-6.30496955e-03,  4.70453911e-02,  9.60909482e-03, ...,
          1.36679364e-02,  2.62902980e-03, -2.45311093e-02],
        [-6.66056061e-03,  1.53017507e-04, -3.15158395e-04, ...,
          1.28758838e-02,  3.71902529e-03,  2.27856319e-02],
        [ 3.69637413e-03, -2.70420294e-02,  2.05683662e-03, ...,
         -1.24421576e-03, -3.74466367e-03, -8.87324940e-03]]],      dtype=float32)
                }
            }
        },
        'x_layers_2': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': Array([-0.00111433,  0.00108239,  0.00224435, ..., -0.00440178,
       -0.00391166,  0.00544218], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[0.9767232, 1.0421209, 1.0658892, ..., 1.0277911, 0.9845215,
        1.040596 ]], dtype=float32),
                        'lora_a': Array([[-0.10845007],
       [ 0.29454732],
       [-0.26034996],
       ...,
       [-0.7982415 ],
       [ 0.7376276 ],
       [-0.08403859]], dtype=float32),
                        'lora_b': Array([[-1.7823721e-05],
       [ 1.4931001e-05],
       [-1.8263425e-05],
       ...,
       [ 7.8286457e-06],
       [ 1.8955880e-05],
       [ 2.3601922e-06]], dtype=float32),
                        'w': Array([[ 0.01403686, -0.05290033,  0.01835991, ...,  0.03456852,
         0.04602428, -0.03192553],
       [-0.02005844, -0.00173637, -0.0569217 , ..., -0.02069917,
        -0.02696633, -0.03619819],
       [-0.04991308, -0.00050655,  0.02636448, ...,  0.00382697,
         0.01514277,  0.03238337],
       ...,
       [-0.01799294, -0.02605661, -0.05084125, ...,  0.0214364 ,
        -0.01124236,  0.03825921],
       [ 0.01654951,  0.0341349 ,  0.00021514, ...,  0.05851978,
         0.00840839, -0.01794812],
       [ 0.01197937,  0.00011841, -0.00112626, ..., -0.03313017,
         0.06482383,  0.03593734]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': Array([ 0.00058029, -0.00185688, -0.00133239, ...,  0.00196204,
       -0.00302488, -0.00129099], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.1031325, 1.1174431, 1.0721775, ..., 1.0805584, 1.0725975,
        1.074459 ]], dtype=float32),
                        'lora_a': Array([[-0.1084796 ],
       [ 0.2945463 ],
       [-0.26038352],
       ...,
       [-0.7982773 ],
       [ 0.7376444 ],
       [-0.08400791]], dtype=float32),
                        'lora_b': Array([[ 5.7603693e-06],
       [ 1.1517750e-05],
       [-1.3028320e-05],
       ...,
       [ 1.4682092e-05],
       [ 1.5251002e-05],
       [ 8.8414801e-08]], dtype=float32),
                        'w': Array([[-0.03089221, -0.02764584,  0.00469999, ...,  0.03527471,
        -0.02542556, -0.03442214],
       [ 0.0074438 ,  0.04577511, -0.00429758, ..., -0.03024394,
        -0.01198478,  0.03910391],
       [ 0.00705164,  0.0343813 , -0.00202623, ..., -0.00482008,
         0.04574559, -0.05240228],
       ...,
       [ 0.00302984, -0.00150592, -0.02762782, ..., -0.04766212,
        -0.01528054,  0.06070133],
       [-0.04403605, -0.02750993, -0.03400619, ...,  0.00910747,
         0.0308539 , -0.01064597],
       [ 0.00639371,  0.04002564, -0.02321222, ...,  0.03084261,
        -0.0078567 , -0.02540698]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': Array([-2.3114951e-03,  6.2909522e-03, -1.1552567e-05, ...,
       -6.4255311e-03, -2.9084997e-03, -8.2080066e-03], dtype=float32),
                    'scale': Array([-0.0221999 , -0.00512881, -0.01245704, ..., -0.01569215,
        0.00904967,  0.00978315], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': Array([1.1226792 , 1.1108161 , 1.0559344 , ..., 0.97350955, 1.0633241 ,
       1.048979  ], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': Array([[ 0.00233258,  0.00309351, -0.00112892, ..., -0.00462134,
        -0.00229245, -0.00666805],
       [-0.00116137,  0.00166665,  0.00024842, ...,  0.000721  ,
         0.00195691,  0.00077325],
       [ 0.00297723,  0.00149911, -0.00517921, ..., -0.00558017,
        -0.00180795,  0.0026498 ],
       ...,
       [-0.00093223, -0.00077007, -0.00018508, ..., -0.00148094,
         0.00050842,  0.0013452 ],
       [-0.00059397, -0.00323179, -0.00278245, ..., -0.00027348,
        -0.00069534,  0.0014734 ],
       [ 0.00202339, -0.00099161,  0.00069069, ..., -0.0009583 ,
         0.00134222, -0.00203341]], dtype=float32),
                    'dora_m': Array([[[0.43050227, 0.3634491 , 0.35903186, ..., 0.37551978,
         0.4474182 , 0.3866278 ],
        [0.40690866, 0.39732417, 0.45042208, ..., 0.41304523,
         0.46480864, 0.40715355],
        [0.36724746, 0.44752336, 0.32034853, ..., 0.33079657,
         0.43708915, 0.38907823],
        ...,
        [0.39494836, 0.37451366, 0.36675626, ..., 0.36984283,
         0.41756925, 0.3461753 ],
        [0.42559335, 0.36781752, 0.39627582, ..., 0.44621998,
         0.43153468, 0.5392652 ],
        [0.39943972, 0.3872124 , 0.5647563 , ..., 0.5671362 ,
         0.3969895 , 0.3899009 ]]], dtype=float32),
                    'lora_a': Array([[-0.1084512 ],
       [ 0.2945504 ],
       [-0.26035422],
       ...,
       [-0.7982447 ],
       [ 0.737619  ],
       [-0.08402732]], dtype=float32),
                    'lora_b': Array([[-8.4393178e-06],
       [ 1.7897872e-05],
       [-4.7108765e-06],
       ...,
       [-1.2980882e-05],
       [-8.7549552e-06],
       [ 9.9082263e-06]], dtype=float32),
                    'w': Array([[[-8.94103758e-03, -1.66658056e-03,  7.67345587e-03, ...,
         -1.61238182e-02,  8.82412028e-03,  6.46348984e-04],
        [ 8.52084626e-03,  6.26490032e-03,  2.17809398e-02, ...,
          2.06468999e-02, -2.45742756e-03, -1.87717024e-02],
        [ 6.54218346e-03, -3.88751812e-02, -1.36708850e-02, ...,
         -1.14361970e-02, -1.76829230e-02,  3.91271226e-02],
        ...,
        [ 1.93273984e-02,  9.61519405e-03,  6.82339817e-03, ...,
         -2.85228714e-02,  9.51640680e-03, -1.33912545e-02],
        [ 2.96449251e-02,  1.76546443e-02, -8.91647674e-03, ...,
         -1.33323688e-02, -1.52229704e-03, -6.13695988e-03],
        [-7.29722390e-03,  1.29208295e-02, -5.45749702e-02, ...,
          2.03211624e-02,  1.15019334e-02,  4.96047409e-03]],

...,

[[-4.51580714e-03,  3.84162646e-03,  1.09713534e-02, ...,
          2.67644849e-04, -1.08704967e-02,  1.14232721e-02],
        [ 4.71198326e-03, -4.28085541e-03, -3.10961213e-02, ...,
          1.92970410e-02, -2.13259384e-02, -1.11752655e-02],
        [ 5.23858657e-03,  1.51326796e-02, -6.44034473e-03, ...,
         -1.95504352e-02,  1.60150714e-02,  5.29391272e-03],
        ...,
        [ 1.52046224e-02, -1.51269860e-03, -1.85855143e-02, ...,
          4.54664137e-03, -1.46121706e-03,  1.15908124e-02],
        [-6.32000528e-03,  2.31103622e-03, -7.22707063e-03, ...,
         -8.15302972e-03, -7.59101938e-03,  5.36400476e-04],
        [-1.05950972e-02, -1.67034008e-02, -6.69758208e-03, ...,
         -2.19507534e-02, -1.03131877e-02, -1.33801838e-02]]],      dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': Array([0.07606351, 0.09519099, 0.08543134, 0.07938089, 0.0478035 ,
       0.05220166, 0.06712113, 0.09687654, 0.07199746, 0.07111655,
       0.09450778, 0.07551523, 0.09265774, 0.08629529, 0.06811125,
       0.07595345, 0.05623099, 0.10769361, 0.07316846, 0.0918607 ,
       0.06741217, 0.06795789, 0.07176007, 0.11473613, 0.06057035,
       0.09941982, 0.10887659, 0.06334271, 0.07600556, 0.11524919,
       0.06202571, 0.0708086 , 0.08118585, 0.06687053, 0.07179871,
       0.07070601, 0.08140425, 0.05987317, 0.06119032, 0.06488035,
       0.08756668, 0.06965479, 0.08641811, 0.07494207, 0.04722185,
       0.10145788, 0.05890552, 0.07864748, 0.06123484, 0.08199554,
       0.10383144, 0.07069265, 0.07977193, 0.08119627, 0.07907593,
       0.06888696, 0.05957579, 0.08070772, 0.06117062, 0.08973919,
       0.06759793, 0.06796879, 0.07075723, 0.08978229, 0.09884267,
       0.06895956, 0.08543922, 0.05556408, 0.08464553, 0.09942926,
       0.07774689, 0.10355261, 0.05608208, 0.09083615, 0.07849273,
       0.06128662, 0.06015168, 0.07853277, 0.06790688, 0.0867492 ],      dtype=float32)
                },
                'post': {
                    'b': Array([-0.00012291,  0.0009653 ,  0.00115237, ...,  0.0021399 ,
       -0.00306339, -0.0042207 ], dtype=float32),
                    'dora_m': Array([[[0.29023057, 0.30365306, 0.3277101 , ..., 0.29610604,
         0.2802406 , 0.26903167],
        [0.31538096, 0.318131  , 0.31961057, ..., 0.31267765,
         0.31972748, 0.3283167 ],
        [0.272802  , 0.3186034 , 0.30230483, ..., 0.29774255,
         0.3166924 , 0.32408988],
        ...,
        [0.3044856 , 0.29973033, 0.2844741 , ..., 0.30133054,
         0.3224635 , 0.29225907],
        [0.30002016, 0.2987829 , 0.29623532, ..., 0.3013424 ,
         0.28689373, 0.2922521 ],
        [0.34111038, 0.32604653, 0.32138798, ..., 0.31741753,
         0.3112618 , 0.29589057]]], dtype=float32),
                    'lora_a': Array([[-0.10848729],
       [ 0.29454032],
       [-0.26038155],
       ...,
       [-0.7982473 ],
       [ 0.7376198 ],
       [-0.08402279]], dtype=float32),
                    'lora_b': Array([[-5.1722172e-06],
       [ 1.1635681e-05],
       [-2.1801025e-05],
       ...,
       [ 1.5885902e-05],
       [ 3.6185668e-06],
       [-9.4091456e-06]], dtype=float32),
                    'w': Array([[[-2.10217144e-02, -3.62480315e-03,  3.03629786e-04, ...,
         -2.47539440e-03, -1.93413980e-02,  3.34993005e-03],
        [-6.61893794e-03,  3.10939061e-03,  9.66002047e-03, ...,
         -1.17761467e-03, -8.80339276e-03, -8.20292626e-03],
        [ 6.34120964e-03, -1.48878177e-03, -2.88486597e-03, ...,
         -4.09498299e-03,  1.14559764e-02, -6.91710680e-04],
        ...,
        [-1.47049502e-02, -1.91921054e-03,  8.64934456e-03, ...,
          1.05494987e-02,  1.48106785e-02, -1.19451089e-02],
        [-9.74751194e-04,  2.43542204e-03,  2.81845173e-03, ...,
          1.61743090e-02, -1.79711059e-02, -7.46698212e-03],
        [-1.45473406e-02, -6.92996383e-03, -2.45343242e-02, ...,
          2.12191343e-02,  5.92035428e-03,  2.15310678e-02]],

...,

[[ 1.29155982e-02, -3.87949613e-03, -1.73111539e-02, ...,
          7.87006505e-03,  1.79969054e-03, -1.99378282e-03],
        [-5.64360945e-03,  6.23272732e-03, -1.77104597e-03, ...,
         -6.94065122e-03, -2.06026528e-03,  2.52026925e-03],
        [ 1.09358048e-02,  1.07644992e-02,  5.83288679e-03, ...,
         -3.34625063e-03,  5.10147365e-04, -4.78237215e-03],
        ...,
        [-4.59554838e-04, -4.96668136e-03,  3.04644182e-03, ...,
         -3.48758954e-03,  1.03788963e-02,  3.95182613e-03],
        [ 4.34665894e-03,  4.57208185e-03, -8.29102844e-03, ...,
          3.36311455e-03, -5.28571196e-03, -2.16717320e-03],
        [ 3.87410005e-03,  2.15103012e-03,  2.00013909e-02, ...,
         -2.00639721e-02, -5.61230956e-03, -1.40962899e-02]]],      dtype=float32)
                },
                'query': {
                    'b': Array([[-0.00189125,  0.01352118,  0.00091355, ...,  0.0035183 ,
         0.00300467, -0.00666348],
       [-0.01087789,  0.02276341,  0.01180435, ...,  0.01987527,
         0.00734986, -0.03836636],
       [-0.00096058,  0.02366084,  0.03013458, ...,  0.02965958,
         0.03138456, -0.0436088 ],
       ...,
       [-0.00894505, -0.03361411,  0.03389145, ...,  0.03919084,
        -0.04814198,  0.0503398 ],
       [-0.02525179, -0.03671117, -0.0318046 , ...,  0.02517176,
         0.03593894, -0.01954985],
       [ 0.01204826, -0.02267273,  0.00109456, ..., -0.0036287 ,
         0.02977114, -0.040389  ]], dtype=float32),
                    'dora_m': Array([[[0.4282941 , 0.41300625, 0.42221516, ..., 0.41875702,
         0.3830318 , 0.4408591 ],
        [0.51572615, 0.54331654, 0.4532205 , ..., 0.4636392 ,
         0.5112157 , 0.4334642 ],
        [0.458033  , 0.47113675, 0.38262296, ..., 0.40013197,
         0.5078299 , 0.45447123],
        ...,
        [0.4023559 , 0.48818073, 0.40680006, ..., 0.47694534,
         0.45200405, 0.48078457],
        [0.5430782 , 0.4305066 , 0.522621  , ..., 0.4887112 ,
         0.6043929 , 0.62397975],
        [0.50998366, 0.608314  , 0.5999261 , ..., 0.55215836,
         0.42419264, 0.66337085]]], dtype=float32),
                    'lora_a': Array([[-0.10845666],
       [ 0.2945355 ],
       [-0.2603558 ],
       ...,
       [-0.798257  ],
       [ 0.7376232 ],
       [-0.08403368]], dtype=float32),
                    'lora_b': Array([[ 1.1979880e-05],
       [ 5.0081144e-06],
       [ 1.1792616e-05],
       ...,
       [ 2.0903781e-05],
       [ 2.0933379e-05],
       [-1.3844640e-05]], dtype=float32),
                    'w': Array([[[ 0.03751517, -0.00894794,  0.03796802, ...,  0.01834398,
          0.01501814, -0.01450621],
        [-0.01260894,  0.0070061 ,  0.01110138, ...,  0.013768  ,
         -0.02294332,  0.00360698],
        [ 0.00125167, -0.00402384,  0.00394119, ...,  0.00744796,
         -0.01517608,  0.00141339],
        ...,
        [-0.02915503,  0.03220129, -0.01296884, ..., -0.00667168,
         -0.00322573, -0.00568456],
        [ 0.00060752,  0.0221884 , -0.01554927, ...,  0.01631006,
          0.03953896, -0.02939055],
        [ 0.04179224, -0.02085465,  0.06698853, ..., -0.05321731,
          0.00048756, -0.06879911]],

...,

[[ 0.01030037,  0.0047197 , -0.01638303, ..., -0.00018055,
          0.00406381, -0.00613182],
        [-0.00418899,  0.01524503,  0.01673017, ...,  0.01751116,
          0.0052884 , -0.03013111],
        [ 0.00471848,  0.00378889, -0.01872631, ..., -0.00934308,
          0.00240882,  0.00095373],
        ...,
        [ 0.00183086,  0.00629113, -0.00941781, ..., -0.01260495,
          0.01857397, -0.01199472],
        [-0.02759313, -0.00985234, -0.0257144 , ..., -0.00575151,
          0.01219597,  0.0021461 ],
        [ 0.0070406 ,  0.00050883, -0.00333748, ...,  0.000177  ,
         -0.00105772, -0.02623683]]], dtype=float32)
                },
                'value': {
                    'b': Array([[ 0.00036897, -0.00305424, -0.00165733, ...,  0.00438358,
         0.00245094, -0.00461676],
       [-0.00301459, -0.00046264,  0.00285859, ...,  0.00075003,
        -0.00377895, -0.00295171],
       [ 0.00346862, -0.0006464 ,  0.00226621, ..., -0.00150375,
         0.00315236, -0.00238282],
       ...,
       [-0.01638476, -0.01227442,  0.01256817, ...,  0.0081502 ,
        -0.00806242, -0.01211064],
       [-0.00075014,  0.0007899 , -0.00100226, ...,  0.00551138,
         0.00104468,  0.00104668],
       [-0.00163955,  0.00094343, -0.00013186, ...,  0.00078494,
         0.00384166,  0.0019375 ]], dtype=float32),
                    'dora_m': Array([[[0.3420103 , 0.32975268, 0.40946823, ..., 0.35417157,
         0.30883542, 0.28574696],
        [0.38379177, 0.39864522, 0.37659323, ..., 0.35450992,
         0.3734042 , 0.40332368],
        [0.31313047, 0.38787195, 0.34699437, ..., 0.33737585,
         0.3530416 , 0.40315   ],
        ...,
        [0.34967658, 0.3342977 , 0.31916252, ..., 0.3604033 ,
         0.3996306 , 0.35014516],
        [0.36047155, 0.3559141 , 0.33937588, ..., 0.37000778,
         0.2856863 , 0.3493786 ],
        [0.4014094 , 0.40934947, 0.39833686, ..., 0.3132786 ,
         0.3536313 , 0.34156358]]], dtype=float32),
                    'lora_a': Array([[-0.10848711],
       [ 0.29451472],
       [-0.26035133],
       ...,
       [-0.7982431 ],
       [ 0.73762923],
       [-0.08403894]], dtype=float32),
                    'lora_b': Array([[-1.2399070e-05],
       [-2.0091564e-05],
       [-2.1121643e-06],
       ...,
       [-6.0697430e-06],
       [ 1.7311204e-05],
       [-1.2977899e-06]], dtype=float32),
                    'w': Array([[[ 0.02118528,  0.02677314, -0.01200122, ...,  0.0262476 ,
          0.00594873, -0.02412984],
        [-0.00146269,  0.00968919, -0.03307377, ..., -0.00141116,
          0.01191948,  0.0207875 ],
        [-0.00940079, -0.00820884, -0.01442705, ...,  0.0038729 ,
         -0.03221527,  0.00978253],
        ...,
        [-0.0257416 , -0.01623189,  0.00299609, ...,  0.01148807,
         -0.00362875,  0.00621605],
        [-0.00116569, -0.02838015,  0.00101068, ...,  0.0427529 ,
          0.01070582, -0.0075488 ],
        [-0.00917997,  0.01766798,  0.00992273, ..., -0.00431697,
          0.02878406,  0.00446529]],

...,

[[-0.02139034, -0.00867905,  0.0218287 , ..., -0.00269548,
         -0.01127221,  0.00096784],
        [ 0.00528475, -0.02709212,  0.00862739, ..., -0.01200321,
         -0.00751034,  0.01631586],
        [-0.01530002,  0.0075484 , -0.00682098, ..., -0.01271224,
         -0.00671244, -0.01073114],
        ...,
        [ 0.01245718, -0.00765948,  0.01388553, ..., -0.02158968,
          0.01284235, -0.00028408],
        [-0.00754987, -0.01706584, -0.00257025, ...,  0.00908255,
          0.0108436 ,  0.00761262],
        [ 0.00830173, -0.00970829, -0.00097407, ...,  0.02274511,
          0.01576199,  0.01823052]]], dtype=float32)
                }
            }
        },
        'x_layers_3': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': Array([-0.00359166, -0.00079451, -0.00558264, ..., -0.00439165,
       -0.0099638 , -0.00504463], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.1016173, 1.5102721, 1.0762283, ..., 0.9510299, 1.0553038,
        1.1028855]], dtype=float32),
                        'lora_a': Array([[-0.10845627],
       [ 0.29451692],
       [-0.26035035],
       ...,
       [-0.7982537 ],
       [ 0.7376246 ],
       [-0.08404098]], dtype=float32),
                        'lora_b': Array([[-1.41747105e-05],
       [ 0.00000000e+00],
       [ 8.15005660e-06],
       ...,
       [ 2.15267755e-05],
       [ 2.56603312e-06],
       [-2.86253481e-07]], dtype=float32),
                        'w': Array([[-0.06025301, -0.03328411,  0.03951382, ...,  0.03327454,
        -0.0440207 , -0.0191193 ],
       [ 0.01228818,  0.06975493,  0.01588618, ...,  0.02443731,
         0.00978233, -0.03454997],
       [-0.0288316 ,  0.00868899, -0.00439149, ..., -0.03457052,
        -0.01619829,  0.00575076],
       ...,
       [ 0.04839077,  0.01288755, -0.04040444, ...,  0.00701007,
        -0.01534467,  0.05396933],
       [ 0.02751558, -0.05551632,  0.01889827, ..., -0.01203401,
        -0.04030662, -0.02222754],
       [-0.03396595, -0.00715171,  0.02327061, ..., -0.02334552,
         0.03825033, -0.01363626]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': Array([ 0.00069713, -0.00100389, -0.0011128 , ...,  0.00128676,
        0.00130776, -0.00264015], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.0751143, 1.1607252, 1.0845897, ..., 1.0613679, 1.1038879,
        1.0690577]], dtype=float32),
                        'lora_a': Array([[-0.10848521],
       [ 0.29453436],
       [-0.26037586],
       ...,
       [-0.798236  ],
       [ 0.73765284],
       [-0.08402206]], dtype=float32),
                        'lora_b': Array([[-3.9151273e-06],
       [-6.3721855e-06],
       [ 9.5884561e-06],
       ...,
       [-1.6667225e-05],
       [ 5.6575900e-06],
       [-6.6767184e-06]], dtype=float32),
                        'w': Array([[ 0.04697682, -0.02177855,  0.0434398 , ...,  0.00020704,
         0.0306547 , -0.02656312],
       [ 0.05644013,  0.03143594, -0.03181333, ..., -0.00783562,
         0.06982915, -0.01424309],
       [ 0.01412346, -0.01519689,  0.02326811, ..., -0.00681298,
        -0.00771268,  0.01557296],
       ...,
       [ 0.04761029, -0.02560747, -0.0354092 , ..., -0.0507902 ,
         0.00030733,  0.00124969],
       [ 0.05086803,  0.00317969, -0.01772834, ..., -0.02629702,
         0.03743565, -0.03532964],
       [-0.03264811,  0.0267511 , -0.04038355, ..., -0.01169836,
        -0.030372  ,  0.0573629 ]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': Array([ 0.00047155,  0.00078917, -0.00076015, ..., -0.00538683,
       -0.00738913,  0.0017588 ], dtype=float32),
                    'scale': Array([-0.01322251, -0.02040597, -0.01582962, ..., -0.00986545,
       -0.00596192,  0.00256199], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': Array([1.0655224, 1.1028657, 1.013484 , ..., 1.0014967, 1.0687963,
       1.0570511], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': Array([[-1.6785270e-03,  1.3397117e-03, -1.0215957e-03, ...,
         5.7368824e-04,  1.0545085e-03, -3.7307583e-03],
       [-1.0163713e-03, -9.7007185e-05,  6.2050847e-03, ...,
         2.6493550e-03,  2.1892705e-03, -2.9838490e-03],
       [-1.2241573e-03,  6.9353578e-04, -2.3339754e-03, ...,
        -3.0547809e-03, -6.9943204e-04,  1.5564183e-04],
       ...,
       [-6.0133091e-03,  6.9217617e-03,  6.3362829e-03, ...,
         2.1950218e-04, -1.9627574e-03, -5.8280532e-03],
       [-1.9570033e-03,  7.5407821e-04,  2.3321738e-04, ...,
        -3.1344227e-06, -1.0981299e-03, -4.7900848e-04],
       [-1.5198577e-03,  8.5386733e-04, -4.1893954e-04, ...,
         3.1281132e-04, -7.4202148e-04,  4.6884449e-04]], dtype=float32),
                    'dora_m': Array([[[0.40797433, 0.4098534 , 0.43326598, ..., 0.38659358,
         0.4861759 , 0.36846244],
        [0.4002248 , 0.4326123 , 0.366221  , ..., 0.38675794,
         0.40489066, 0.41617996],
        [0.5055748 , 0.54978096, 0.44103009, ..., 0.49361414,
         0.44243976, 0.6074491 ],
        ...,
        [0.33700776, 0.36508954, 0.30899486, ..., 0.34763682,
         0.49400184, 0.31611404],
        [0.47811446, 0.3784983 , 0.5255163 , ..., 0.39022654,
         0.4343925 , 0.4930178 ],
        [0.5566264 , 0.46944293, 0.45579225, ..., 0.520067  ,
         0.55533904, 0.4790281 ]]], dtype=float32),
                    'lora_a': Array([[-0.10845666],
       [ 0.29454505],
       [-0.2603801 ],
       ...,
       [-0.79825234],
       [ 0.7376466 ],
       [-0.08400684]], dtype=float32),
                    'lora_b': Array([[-1.0591593e-05],
       [-2.0337413e-06],
       [ 7.4934142e-06],
       ...,
       [ 1.5904568e-05],
       [-1.3928719e-05],
       [-7.3558276e-06]], dtype=float32),
                    'w': Array([[[ 6.0837939e-03,  4.6521290e-03,  7.5706211e-04, ...,
          9.1714406e-04, -3.6846860e-03, -5.4185386e-03],
        [-6.0345409e-03,  2.2672977e-02,  8.0101583e-03, ...,
         -5.0341138e-03, -2.5518978e-02,  1.1301177e-02],
        [ 1.6000951e-03,  1.6028784e-02,  4.7602216e-03, ...,
          4.7910311e-03,  1.6524794e-02, -2.1223988e-02],
        ...,
        [-3.4377207e-03,  3.5684025e-03,  8.2413079e-03, ...,
          9.2745209e-03,  1.6370324e-02, -9.4568986e-04],
        [ 1.3595778e-03,  1.3690447e-03,  2.4898786e-02, ...,
          5.3510466e-03, -2.4402350e-05, -1.7150324e-02],
        [-2.2716202e-02,  1.7809145e-02, -9.5531344e-03, ...,
          8.5292039e-03,  2.0799288e-03,  1.3175344e-02]],

...,

[[ 5.9171738e-03, -4.3242317e-04, -3.3961914e-02, ...,
         -2.6869291e-02, -1.5355121e-05,  7.3913564e-03],
        [-1.7731775e-03, -3.9709383e-03,  3.4796234e-04, ...,
         -1.1495024e-03,  4.7223768e-03,  1.2998351e-02],
        [-1.9058391e-02, -1.8019665e-02,  1.3039748e-02, ...,
          1.6119709e-03, -5.7866625e-03, -6.1591226e-03],
        ...,
        [-1.1477595e-02, -6.9808867e-04,  6.9915270e-03, ...,
         -1.6907848e-02,  2.3044579e-02, -8.4827431e-03],
        [-1.1313060e-02,  3.2085020e-04, -5.4306183e-05, ...,
         -6.9154515e-03, -1.3521667e-02,  1.3901893e-03],
        [ 1.0984845e-03,  1.9725360e-02, -1.0533265e-02, ...,
         -2.8858619e-04, -1.2176596e-02, -1.1769810e-03]]], dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': Array([0.08037373, 0.07929973, 0.05286393, 0.12249207, 0.07311964,
       0.06011745, 0.06764143, 0.08867034, 0.07810722, 0.11073345,
       0.10193466, 0.05869003, 0.08364506, 0.07253328, 0.0851353 ,
       0.0989499 , 0.06423612, 0.06726725, 0.06934468, 0.06810744,
       0.08061647, 0.06345143, 0.06148439, 0.08881178, 0.09636571,
       0.07103957, 0.07336371, 0.09454832, 0.07582602, 0.12069155,
       0.07495704, 0.05750223, 0.09781589, 0.10714186, 0.06152625,
       0.11027586, 0.07866532, 0.13079058, 0.10691793, 0.13633348,
       0.0731928 , 0.05952192, 0.0733497 , 0.07639648, 0.05115701,
       0.0510252 , 0.09616829, 0.07447171, 0.05894472, 0.05770541,
       0.10170057, 0.05173   , 0.08556115, 0.08349261, 0.08772631,
       0.05666647, 0.05503659, 0.07228909, 0.08734103, 0.06599794,
       0.07190575, 0.06675142, 0.12766062, 0.07212538, 0.06819951,
       0.0679538 , 0.05753606, 0.11283694, 0.11276623, 0.07136308,
       0.06852216, 0.07107771, 0.07141783, 0.07693509, 0.09175616,
       0.05311742, 0.0721948 , 0.05768591, 0.08882511, 0.0579024 ],      dtype=float32)
                },
                'post': {
                    'b': Array([ 0.00054931, -0.00043755,  0.00115337, ...,  0.00313751,
        0.00021384, -0.00234471], dtype=float32),
                    'dora_m': Array([[[0.34412032, 0.33978143, 0.3351247 , ..., 0.34741372,
         0.37539455, 0.3355618 ],
        [0.3671442 , 0.31677768, 0.3082428 , ..., 0.3351626 ,
         0.31704226, 0.35033002],
        [0.3527745 , 0.36563784, 0.3297209 , ..., 0.33538303,
         0.33380273, 0.33877623],
        ...,
        [0.32376668, 0.291418  , 0.3045238 , ..., 0.306671  ,
         0.30240694, 0.2946346 ],
        [0.388179  , 0.38386187, 0.40650725, ..., 0.43480322,
         0.35135937, 0.36934292],
        [0.32978594, 0.33507746, 0.35941535, ..., 0.37659055,
         0.34447354, 0.33983544]]], dtype=float32),
                    'lora_a': Array([[-0.10848659],
       [ 0.2945429 ],
       [-0.26038474],
       ...,
       [-0.79824585],
       [ 0.73764515],
       [-0.08402764]], dtype=float32),
                    'lora_b': Array([[ 1.2616821e-05],
       [-1.6618176e-05],
       [-1.1939699e-05],
       ...,
       [-1.9479297e-05],
       [ 1.7031291e-05],
       [ 1.1059015e-07]], dtype=float32),
                    'w': Array([[[-1.54192438e-02,  1.00835366e-02, -5.50007075e-03, ...,
          1.30757838e-02,  3.85335903e-03,  8.46714713e-03],
        [ 1.19337235e-02, -7.40378397e-03, -8.98601767e-03, ...,
          1.74418994e-04,  9.26105119e-03,  1.36058349e-02],
        [-1.26508959e-02, -1.19769142e-03,  2.10273359e-02, ...,
         -1.45233786e-02, -1.35016337e-03,  1.32047795e-02],
        ...,
        [-2.17427174e-03,  3.66755878e-04, -1.46679357e-02, ...,
          1.28410952e-02,  1.63965933e-02,  1.18775107e-03],
        [-1.10554257e-02, -1.21107725e-02, -1.38095804e-02, ...,
         -4.13777167e-03,  2.90608988e-03,  1.77041776e-02],
        [ 6.82405941e-03,  2.09691795e-03,  4.36124858e-03, ...,
         -1.91892106e-02,  1.02002954e-03, -1.25868535e-02]],

...,

[[ 2.22686562e-03,  2.18903972e-03,  1.50927959e-03, ...,
          1.07232034e-02, -1.32909827e-02,  1.15348352e-02],
        [ 1.62409302e-02, -2.02017254e-03, -1.52577711e-02, ...,
          3.22888186e-03, -2.56273290e-03, -7.44445203e-03],
        [-4.24485304e-04, -1.38442023e-02,  2.75478279e-03, ...,
          4.94189747e-03,  4.66359500e-03,  2.97783059e-03],
        ...,
        [-8.09659809e-03, -1.92497522e-02, -1.40441665e-02, ...,
          5.59709780e-03, -1.26869930e-03,  2.40132771e-03],
        [-8.65959469e-03,  1.91764440e-03, -1.21536665e-02, ...,
         -5.53380745e-03,  9.27343644e-05, -6.86470233e-03],
        [ 2.26667942e-03, -8.99902545e-03, -1.91209503e-02, ...,
         -7.43969483e-03, -2.70407391e-03,  1.19197704e-02]]],      dtype=float32)
                },
                'query': {
                    'b': Array([[-0.0063142 , -0.00455809,  0.02359539, ...,  0.03004604,
        -0.01363834, -0.01749385],
       [-0.00883258,  0.01348464, -0.01877279, ..., -0.00789092,
        -0.00137584,  0.00400858],
       [ 0.00920692,  0.00321167,  0.01175659, ...,  0.00476064,
        -0.00365732,  0.00989715],
       ...,
       [-0.02695556,  0.02235626,  0.01659154, ..., -0.02419706,
        -0.00562116, -0.02516452],
       [ 0.03666452,  0.03104526,  0.03207183, ..., -0.01275486,
        -0.00320612, -0.01301267],
       [ 0.00257295,  0.0046639 , -0.01591201, ..., -0.01375522,
         0.03187206, -0.00602493]], dtype=float32),
                    'dora_m': Array([[[0.5695813 , 0.52736235, 0.58710164, ..., 0.51412064,
         0.58267385, 0.5494198 ],
        [0.4271132 , 0.40858507, 0.4741437 , ..., 0.49109793,
         0.41087383, 0.46641636],
        [0.5951725 , 0.6253242 , 0.42670807, ..., 0.5321043 ,
         0.51813567, 0.69371593],
        ...,
        [0.44667596, 0.43793243, 0.36729166, ..., 0.42633805,
         0.4833749 , 0.39590195],
        [0.82649726, 0.695414  , 0.6932399 , ..., 0.64766765,
         0.55562294, 0.58919406],
        [0.62248015, 0.6545785 , 0.686005  , ..., 0.55704916,
         0.70983607, 0.62033194]]], dtype=float32),
                    'lora_a': Array([[-0.10846189],
       [ 0.2945191 ],
       [-0.26035792],
       ...,
       [-0.7982664 ],
       [ 0.73763984],
       [-0.08402401]], dtype=float32),
                    'lora_b': Array([[-1.0752186e-05],
       [-1.5510323e-05],
       [-1.0249877e-05],
       ...,
       [ 1.7887916e-05],
       [ 1.6094427e-05],
       [-2.6606815e-06]], dtype=float32),
                    'w': Array([[[ 1.65672582e-02,  3.58093791e-02,  1.30947670e-02, ...,
          1.43328891e-03,  4.70675714e-02,  2.24824739e-03],
        [-1.48226786e-02,  1.66079383e-02, -3.66222626e-03, ...,
          7.23349676e-03, -1.60129054e-03, -1.37269692e-02],
        [-1.83571000e-02,  3.74624468e-02,  7.09347241e-03, ...,
         -5.79345413e-02,  2.28687190e-03, -6.03809953e-03],
        ...,
        [ 4.69985092e-03,  6.11597765e-03,  8.14028841e-04, ...,
         -5.70318429e-03,  1.45086637e-02, -8.69368576e-03],
        [ 8.65209382e-03,  1.80686526e-02,  3.40784229e-02, ...,
         -6.34136191e-03, -4.05705571e-02,  2.96484702e-03],
        [-1.02002937e-02,  1.63281299e-02,  2.41871784e-03, ...,
          4.32097306e-03,  1.67190302e-02, -2.59064417e-02]],

...,

[[-5.17609809e-03,  2.61216294e-02, -7.79005094e-03, ...,
         -1.43469609e-02, -1.07553983e-02, -4.63652145e-03],
        [ 1.22595262e-02, -1.52036808e-02, -1.28153181e-02, ...,
          1.21101914e-02,  8.61967448e-03, -1.13262469e-02],
        [-1.66602302e-02, -1.14014233e-02, -1.76152457e-02, ...,
         -9.42711439e-03,  8.81423708e-03, -6.95279287e-03],
        ...,
        [-8.93063005e-03,  1.06965024e-02,  5.01304539e-03, ...,
         -5.93680236e-03, -1.38604939e-02, -1.03369784e-02],
        [ 1.32499551e-02,  9.15231463e-03,  7.31133949e-03, ...,
         -1.11583835e-02,  5.18290140e-03,  9.23759025e-03],
        [-5.59666334e-03, -6.71564310e-04,  1.70062017e-02, ...,
          3.26499939e-02,  2.08929162e-02, -8.61088000e-03]]],      dtype=float32)
                },
                'value': {
                    'b': Array([[ 5.5404363e-04, -1.6793143e-03, -9.0165624e-05, ...,
        -6.5793184e-04, -1.2384902e-03,  9.3860185e-04],
       [ 2.4099937e-03, -6.8471888e-03,  6.3015427e-03, ...,
        -3.8942320e-03,  6.0042161e-03,  1.4533896e-03],
       [ 3.7580477e-03, -1.9677782e-03, -4.0561365e-04, ...,
         5.9572086e-03,  7.6286710e-04,  5.6843081e-04],
       ...,
       [ 2.3894291e-03,  2.4938518e-03, -2.3374441e-03, ...,
        -5.7953368e-03, -4.7948356e-03, -3.9126086e-03],
       [-2.7866389e-03, -2.4954984e-03, -1.5062568e-03, ...,
         3.5421588e-04, -5.6915311e-04,  1.1925710e-03],
       [ 2.6903339e-03,  2.7697408e-03,  1.0172953e-04, ...,
        -1.0963748e-04,  2.3099105e-03, -9.5514680e-04]], dtype=float32),
                    'dora_m': Array([[[0.4142267 , 0.39498794, 0.40760034, ..., 0.418609  ,
         0.45753276, 0.38569057],
        [0.44152302, 0.35415035, 0.3573696 , ..., 0.40267357,
         0.3883374 , 0.44105795],
        [0.4047406 , 0.44545203, 0.40715402, ..., 0.40127373,
         0.40111724, 0.4081952 ],
        ...,
        [0.40549988, 0.35667303, 0.37096703, ..., 0.3815067 ,
         0.37163144, 0.3547481 ],
        [0.48809615, 0.44063643, 0.499354  , ..., 0.5165337 ,
         0.4245573 , 0.43074387],
        [0.37428564, 0.38100007, 0.44260135, ..., 0.47488302,
         0.41578785, 0.41935116]]], dtype=float32),
                    'lora_a': Array([[-0.10848758],
       [ 0.29451287],
       [-0.26035085],
       ...,
       [-0.7982473 ],
       [ 0.73763263],
       [-0.08404037]], dtype=float32),
                    'lora_b': Array([[-1.6103431e-05],
       [-9.2413056e-06],
       [ 1.3052653e-05],
       ...,
       [-1.9332869e-05],
       [-1.4518726e-05],
       [-8.1482676e-07]], dtype=float32),
                    'w': Array([[[-7.58182677e-03, -3.36851273e-03,  3.39671485e-02, ...,
          9.98453703e-03, -4.34027612e-03, -2.52746092e-03],
        [-1.20935272e-02,  2.24267375e-02,  7.97357119e-04, ...,
         -8.71001463e-03, -1.41596375e-02, -1.06950598e-02],
        [ 3.16537847e-03, -1.30682709e-02, -1.19612925e-02, ...,
         -7.02873431e-03,  3.03701207e-04,  5.01969084e-03],
        ...,
        [ 2.14373181e-03, -1.49115967e-02, -9.14988853e-03, ...,
         -3.12511139e-02,  7.74393207e-04, -9.39847156e-03],
        [ 2.83262059e-02, -1.54163834e-04,  7.94346852e-04, ...,
          2.94039142e-03, -2.18230039e-02, -2.59777851e-04],
        [ 1.26834419e-02,  2.08839588e-02,  1.99626083e-03, ...,
          1.73709262e-02, -1.05466777e-02, -2.94377445e-03]],

...,

[[ 2.23035496e-02, -6.43958477e-03,  1.02769854e-02, ...,
         -5.03790658e-03,  1.96808223e-02, -3.42780426e-02],
        [-1.35366600e-02,  4.95954510e-03,  1.61573887e-02, ...,
          9.64686275e-03,  2.81135610e-04, -7.91399926e-03],
        [-2.77084392e-03,  5.36162453e-03,  1.09172724e-02, ...,
         -4.54607280e-03, -1.73717504e-03, -6.74237777e-03],
        ...,
        [ 3.01251691e-02,  1.58525016e-02,  1.94437802e-02, ...,
          1.11673763e-02,  9.36414767e-03, -2.37105903e-03],
        [ 1.60713494e-02, -5.39772771e-03,  1.84250027e-02, ...,
         -1.80582572e-02, -5.87218814e-03,  2.32455432e-02],
        [-4.67143068e-03, -9.34725441e-03,  5.61833940e-03, ...,
          5.75147616e-03,  1.40718175e-02,  2.38939351e-03]]],      dtype=float32)
                }
            }
        },
        'x_layers_4': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': Array([-0.00033918, -0.00515734, -0.00025558, ...,  0.00321242,
       -0.0045813 ,  0.00603206], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.0330225, 1.0914443, 1.1513537, ..., 1.0884908, 1.0164511,
        1.0796851]], dtype=float32),
                        'lora_a': Array([[-0.10848471],
       [ 0.29452944],
       [-0.26035035],
       ...,
       [-0.79827106],
       [ 0.737649  ],
       [-0.08404028]], dtype=float32),
                        'lora_b': Array([[ 1.6508468e-05],
       [-1.9674018e-05],
       [ 1.0563112e-05],
       ...,
       [-1.3691221e-06],
       [-4.3420587e-06],
       [ 2.0136307e-05]], dtype=float32),
                        'w': Array([[ 0.00432622, -0.03287012,  0.01275862, ...,  0.03733737,
        -0.00666521, -0.01324154],
       [-0.03115523,  0.03335258,  0.01877301, ...,  0.03141083,
        -0.00394512, -0.05304177],
       [ 0.00881899, -0.05999147,  0.00512024, ...,  0.01997933,
         0.04666889, -0.02512316],
       ...,
       [ 0.01407511, -0.01288593, -0.01321636, ...,  0.04786241,
         0.00889695,  0.03299772],
       [ 0.00216974,  0.01110496,  0.02895355, ..., -0.04200781,
         0.02919233,  0.03616085],
       [ 0.02409721, -0.03261668,  0.03869701, ..., -0.01949932,
         0.00740029, -0.02987003]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': Array([ 1.2490508e-03,  4.1453599e-04, -2.8177700e-03, ...,
        1.4040343e-04,  3.5180324e-03, -7.5857452e-06], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.103094 , 1.1773516, 1.1127034, ..., 1.1364249, 1.1226662,
        1.1492687]], dtype=float32),
                        'lora_a': Array([[-0.1084888 ],
       [ 0.29451734],
       [-0.26037604],
       ...,
       [-0.79827285],
       [ 0.73763615],
       [-0.08404085]], dtype=float32),
                        'lora_b': Array([[ 1.7715147e-05],
       [-5.1798033e-06],
       [-1.4958599e-05],
       ...,
       [ 6.6862999e-06],
       [ 1.5821739e-05],
       [ 6.8767995e-06]], dtype=float32),
                        'w': Array([[-0.00270853,  0.007068  ,  0.00870072, ..., -0.04568937,
        -0.00149104, -0.04165388],
       [ 0.05179108, -0.02407867, -0.01176148, ...,  0.03194524,
        -0.02103354, -0.0012801 ],
       [ 0.01410516,  0.03630389,  0.00128203, ...,  0.03379128,
        -0.02607304,  0.04072599],
       ...,
       [ 0.0087957 , -0.01915103, -0.00403893, ...,  0.02046533,
         0.0604092 ,  0.05353087],
       [-0.03155645,  0.04317928,  0.02834019, ..., -0.03319148,
        -0.04890137, -0.02391277],
       [-0.03234796,  0.04266334,  0.01308641, ..., -0.00362083,
        -0.02230242,  0.00610088]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': Array([-0.0006323 , -0.00250285,  0.00583033, ...,  0.00232698,
        0.00085398, -0.00280266], dtype=float32),
                    'scale': Array([-0.01994971,  0.00063523,  0.01054509, ..., -0.01481609,
        0.00990174, -0.01184579], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': Array([1.1022189, 1.11532  , 1.0147704, ..., 1.016388 , 1.0738846,
       1.0579767], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': Array([[-1.8876005e-03, -4.2480972e-04,  3.3034892e-03, ...,
        -6.5590523e-04, -5.9691512e-05,  1.5365365e-03],
       [-1.7957551e-03, -3.6221617e-03,  1.1712223e-03, ...,
        -6.4384378e-04,  1.7770033e-03,  5.6157954e-04],
       [-3.7401831e-03,  9.2523871e-04,  2.3624795e-03, ...,
         1.7385277e-03, -7.7581783e-05,  7.2749582e-04],
       ...,
       [-5.9823133e-03, -1.1551752e-03,  1.7608545e-03, ...,
         2.5742259e-03, -3.2627333e-03, -4.1508870e-03],
       [ 5.9462013e-04, -3.0149426e-05,  2.7025258e-04, ...,
         1.5758632e-03,  1.9301582e-03,  4.0970929e-04],
       [ 8.8718737e-04, -1.5026670e-03,  1.1991300e-03, ...,
        -2.1778243e-03, -1.1430072e-03, -9.6649473e-04]], dtype=float32),
                    'dora_m': Array([[[0.4779925 , 0.5473837 , 0.47067213, ..., 0.40664753,
         0.49535072, 0.40305293],
        [0.44623137, 0.36741653, 0.38836437, ..., 0.4508105 ,
         0.43770865, 0.5159256 ],
        [0.52858734, 0.41964146, 0.5178237 , ..., 0.4401971 ,
         0.43517315, 0.581075  ],
        ...,
        [0.34480128, 0.32434702, 0.3779692 , ..., 0.376078  ,
         0.32489   , 0.3766901 ],
        [0.3988427 , 0.45798433, 0.47958797, ..., 0.5078109 ,
         0.45117748, 0.41140974],
        [0.4690084 , 0.5693717 , 0.5124356 , ..., 0.42941794,
         0.445709  , 0.4846299 ]]], dtype=float32),
                    'lora_a': Array([[-0.10846709],
       [ 0.29454607],
       [-0.2603837 ],
       ...,
       [-0.7982513 ],
       [ 0.73762035],
       [-0.08401654]], dtype=float32),
                    'lora_b': Array([[ 1.7066379e-05],
       [-1.3798385e-05],
       [-2.0225169e-05],
       ...,
       [-2.9385133e-06],
       [-8.9455370e-06],
       [ 1.5834339e-05]], dtype=float32),
                    'w': Array([[[ 3.01337000e-02,  1.24209067e-02, -5.86472964e-03, ...,
         -8.65774229e-03, -1.08871749e-02,  2.49727443e-02],
        [ 2.31124274e-02, -1.10988012e-02,  1.84235850e-03, ...,
         -1.88146941e-02,  3.76610714e-03,  2.84295399e-02],
        [ 3.14491009e-03, -8.37674830e-03, -1.24650598e-02, ...,
         -6.69335388e-03, -4.62858845e-03,  5.41267218e-03],
        ...,
        [ 7.64358323e-03, -8.19855649e-03,  9.58123524e-03, ...,
         -6.77058846e-03, -1.36759533e-02,  6.71977177e-03],
        [ 4.26648557e-03, -9.02947877e-03, -8.26200470e-03, ...,
         -1.04721291e-02,  3.18084680e-03,  5.80366934e-03],
        [-2.57805432e-03,  5.40386792e-03, -1.25376377e-02, ...,
         -1.93149422e-03, -9.40163434e-03,  1.43176261e-02]],

...,

[[ 8.64202995e-03, -1.30942725e-02, -1.28558045e-02, ...,
         -1.35812210e-02,  1.13754417e-04,  1.10555319e-02],
        [-4.71161539e-03,  1.73984803e-02, -1.53125841e-02, ...,
          8.52911081e-03,  8.62509571e-03,  1.41716450e-02],
        [ 5.33405552e-03,  6.14917604e-03, -9.92153399e-03, ...,
          1.73857119e-02, -2.15481967e-02,  5.46277035e-03],
        ...,
        [-2.68412335e-03,  2.79641099e-04, -1.33013939e-02, ...,
          4.02520550e-03, -3.90526606e-03, -5.58624044e-03],
        [ 1.42477348e-03, -3.74906085e-04, -1.70411021e-02, ...,
          8.66159983e-03,  1.14939520e-02,  6.46173663e-04],
        [ 7.69440318e-04,  9.34678968e-03, -1.60742626e-02, ...,
          1.88496392e-02, -6.27175719e-03,  1.66990552e-02]]],      dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': Array([0.06925689, 0.0687267 , 0.09816898, 0.05614004, 0.10127976,
       0.09549069, 0.05992376, 0.1010856 , 0.07627991, 0.06381155,
       0.06614725, 0.11821901, 0.1060421 , 0.09320906, 0.08363871,
       0.09773096, 0.08945961, 0.08148175, 0.08691528, 0.05950036,
       0.1054969 , 0.07629689, 0.06731045, 0.05241217, 0.07037603,
       0.08807229, 0.08101945, 0.11336018, 0.08727027, 0.08233001,
       0.1025634 , 0.10394803, 0.10563974, 0.06789199, 0.15156211,
       0.0974873 , 0.09855369, 0.09362685, 0.08743744, 0.04419754,
       0.09561277, 0.08159087, 0.05990618, 0.12188151, 0.07485054,
       0.1204315 , 0.08618229, 0.09011715, 0.0703643 , 0.11945228,
       0.07089204, 0.10021815, 0.09990682, 0.08731607, 0.08622605,
       0.12677069, 0.10883871, 0.09831861, 0.09507571, 0.06917293,
       0.08881298, 0.10157361, 0.10612229, 0.10519458, 0.12216848,
       0.09660444, 0.10267104, 0.08772461, 0.0829059 , 0.11468416,
       0.08165301, 0.0679692 , 0.07408445, 0.07055859, 0.10889028,
       0.07661044, 0.08274958, 0.09309391, 0.06840506, 0.06962132],      dtype=float32)
                },
                'post': {
                    'b': Array([ 0.00018584,  0.00054333,  0.00215429, ...,  0.00092777,
        0.00437075, -0.00032677], dtype=float32),
                    'dora_m': Array([[[0.3549017 , 0.35065892, 0.3242511 , ..., 0.35521683,
         0.33345848, 0.33935705],
        [0.3304873 , 0.31610116, 0.35941514, ..., 0.31335917,
         0.35120377, 0.3141185 ],
        [0.39504492, 0.35642606, 0.39548331, ..., 0.38269466,
         0.37367317, 0.41942745],
        ...,
        [0.3217648 , 0.35797754, 0.35486513, ..., 0.31420276,
         0.3357392 , 0.31748652],
        [0.34582222, 0.34153035, 0.32220906, ..., 0.32697836,
         0.3472013 , 0.33797365],
        [0.37102264, 0.3736804 , 0.36504176, ..., 0.3352303 ,
         0.34101838, 0.34604383]]], dtype=float32),
                    'lora_a': Array([[-0.10848638],
       [ 0.29454887],
       [-0.2603829 ],
       ...,
       [-0.7982457 ],
       [ 0.7376416 ],
       [-0.08402371]], dtype=float32),
                    'lora_b': Array([[-1.6716027e-05],
       [ 1.6542637e-05],
       [-9.4244797e-06],
       ...,
       [-1.8103772e-05],
       [-9.2715281e-06],
       [ 8.3185269e-06]], dtype=float32),
                    'w': Array([[[-0.00364268, -0.00533502,  0.02635869, ...,  0.00606002,
         -0.00487637, -0.00248202],
        [ 0.01556431,  0.02563867,  0.00874958, ...,  0.01622818,
          0.00538342, -0.00068508],
        [-0.00566908, -0.00174261, -0.00176214, ...,  0.00396739,
          0.01358565, -0.00684316],
        ...,
        [ 0.0211008 ,  0.001731  , -0.01537344, ..., -0.00073908,
          0.01017119, -0.01133754],
        [ 0.00807352, -0.02239311,  0.00981964, ..., -0.02785161,
          0.00785703, -0.0189785 ],
        [-0.0037584 ,  0.00413363, -0.0038328 , ...,  0.010292  ,
          0.00581   , -0.00778716]],

...,

[[-0.00093579,  0.00218967,  0.00322592, ..., -0.00076474,
          0.01170467, -0.00524056],
        [ 0.01144154, -0.00816934,  0.01511669, ..., -0.01021524,
         -0.01007736,  0.00074415],
        [-0.00469364, -0.00903361,  0.02324042, ..., -0.01450897,
          0.00737509, -0.00731732],
        ...,
        [-0.00158206, -0.00973321, -0.01214427, ..., -0.00271589,
         -0.00512956,  0.01256492],
        [-0.00496091, -0.00923559,  0.00735262, ...,  0.00423574,
         -0.01897817,  0.0084137 ],
        [-0.01685073,  0.02206317, -0.00042697, ..., -0.01593289,
         -0.0175844 , -0.01029996]]], dtype=float32)
                },
                'query': {
                    'b': Array([[-0.00904353,  0.00922141,  0.00253093, ...,  0.0071486 ,
         0.01894851, -0.00297219],
       [ 0.02580181, -0.01197189,  0.03771947, ..., -0.01448922,
         0.03808154,  0.0124295 ],
       [-0.00649451, -0.00354858,  0.00604098, ..., -0.00584646,
         0.00274818,  0.00644646],
       ...,
       [ 0.01927792,  0.00616192, -0.01234802, ..., -0.01084663,
         0.02252456,  0.01259941],
       [ 0.01681058, -0.01131948,  0.00902725, ..., -0.00339165,
        -0.0137673 , -0.00382652],
       [ 0.00305724,  0.00126192,  0.00328541, ..., -0.00471952,
         0.0095676 , -0.00190623]], dtype=float32),
                    'dora_m': Array([[[0.58302927, 0.53612065, 0.70577985, ..., 0.6650343 ,
         0.5249793 , 0.53525794],
        [0.6316028 , 0.5070909 , 0.6136609 , ..., 0.5539808 ,
         0.47871527, 0.55615276],
        [0.77479476, 0.6321354 , 0.7036078 , ..., 0.7078841 ,
         0.6686136 , 0.66476214],
        ...,
        [0.42415258, 0.39790225, 0.4772801 , ..., 0.40934038,
         0.32918286, 0.46008918],
        [0.5191779 , 0.6217404 , 0.6219015 , ..., 0.602912  ,
         0.5460744 , 0.46815467],
        [0.69114596, 0.5633315 , 0.6487506 , ..., 0.5861016 ,
         0.696812  , 0.5075813 ]]], dtype=float32),
                    'lora_a': Array([[-0.10848147],
       [ 0.2945203 ],
       [-0.2603582 ],
       ...,
       [-0.7982709 ],
       [ 0.7376243 ],
       [-0.08403166]], dtype=float32),
                    'lora_b': Array([[-1.9305344e-05],
       [ 1.3850207e-05],
       [ 1.3093215e-05],
       ...,
       [ 8.6143145e-06],
       [-1.3614863e-05],
       [ 1.5754793e-05]], dtype=float32),
                    'w': Array([[[ 2.46237386e-02,  4.97382774e-04,  1.69206820e-02, ...,
          2.41074469e-02, -1.38304487e-03,  2.32416531e-03],
        [ 2.64098775e-02, -1.28170568e-02,  4.24231729e-03, ...,
         -1.04854880e-02,  1.35888867e-02,  2.55975388e-02],
        [ 3.81973125e-02, -1.20887049e-02, -4.18927427e-03, ...,
         -3.14308107e-02, -4.72382940e-02, -1.75638311e-02],
        ...,
        [ 2.11419575e-02,  6.90283673e-03, -3.24399546e-02, ...,
         -3.98736959e-03,  1.54951680e-02,  8.00326746e-03],
        [ 3.67254554e-03,  6.81026978e-03, -2.48883180e-02, ...,
         -8.82120710e-03,  2.65567023e-02,  6.48691785e-03],
        [-8.71083350e-04,  1.85516886e-02, -2.39707213e-02, ...,
         -1.15928007e-02,  1.32977040e-02,  1.84124187e-02]],

...,

[[-2.82171890e-02, -2.35241577e-02, -8.98296759e-03, ...,
         -2.17941846e-03,  3.43297906e-02, -2.92673726e-02],
        [-9.73666925e-03,  1.44522614e-03,  2.03186627e-02, ...,
          7.75819179e-03,  6.99952804e-03,  1.75368171e-02],
        [-1.80099886e-02,  1.42530501e-02,  2.05159020e-02, ...,
         -3.35845947e-02,  9.71715711e-03, -2.32642633e-03],
        ...,
        [ 1.70393952e-03,  3.93192488e-04,  1.15964804e-02, ...,
         -4.94584255e-03,  5.23317885e-03,  3.36621935e-03],
        [-2.14738287e-02,  2.95266155e-02,  2.52535124e-03, ...,
         -3.53639126e-02,  1.94096528e-02, -1.27472002e-02],
        [ 8.97306192e-04, -1.37348566e-02,  3.49913090e-02, ...,
         -8.99178721e-03,  5.32816956e-03, -1.90543346e-02]]],      dtype=float32)
                },
                'value': {
                    'b': Array([[ 2.1524420e-03,  2.9411949e-03,  1.0325930e-03, ...,
         3.1009023e-03, -1.1407599e-03, -1.3143895e-04],
       [-1.3207317e-03, -1.9708832e-03, -1.9265555e-03, ...,
         9.8339503e-04, -1.3239905e-03, -9.9254667e-04],
       [-1.7794296e-04,  8.9951250e-04, -1.1194174e-03, ...,
         1.2776342e-03,  6.6736236e-04, -1.1825715e-03],
       ...,
       [ 2.9136413e-03,  2.6662368e-03,  9.9231875e-05, ...,
        -5.4693753e-03,  2.3253520e-03, -2.8927152e-03],
       [-7.2450127e-04,  4.6958555e-03, -1.4273097e-03, ...,
         3.8719655e-03, -1.3461473e-03,  2.1226092e-03],
       [ 2.2172735e-03,  1.1708419e-03, -1.0171951e-03, ...,
         1.0178455e-03, -4.1146367e-04, -2.4446507e-04]], dtype=float32),
                    'dora_m': Array([[[0.40572873, 0.42254555, 0.3886209 , ..., 0.3959129 ,
         0.3948047 , 0.4200979 ],
        [0.39290428, 0.36910173, 0.42487058, ..., 0.38255998,
         0.41265386, 0.36029127],
        [0.4840095 , 0.42997918, 0.4866197 , ..., 0.44483402,
         0.4603386 , 0.498332  ],
        ...,
        [0.38858452, 0.43791658, 0.44983134, ..., 0.3505808 ,
         0.40317747, 0.35174215],
        [0.4126554 , 0.40841237, 0.39503822, ..., 0.4126465 ,
         0.4231076 , 0.40950686],
        [0.44272456, 0.48116374, 0.44691628, ..., 0.40953773,
         0.3994937 , 0.3991045 ]]], dtype=float32),
                    'lora_a': Array([[-0.10849018],
       [ 0.29451308],
       [-0.26035154],
       ...,
       [-0.79827666],
       [ 0.73761666],
       [-0.08404027]], dtype=float32),
                    'lora_b': Array([[-2.2238488e-05],
       [ 1.7289543e-05],
       [ 1.5412583e-05],
       ...,
       [ 1.9027617e-05],
       [ 7.2750495e-06],
       [-6.5653435e-06]], dtype=float32),
                    'w': Array([[[-8.27747304e-03, -2.74438374e-02, -2.21472769e-03, ...,
         -8.86141602e-03, -3.90934292e-03, -8.87976680e-03],
        [-4.07242536e-04, -4.85621393e-03,  4.32445714e-03, ...,
         -1.37868756e-02, -1.85714811e-02, -4.08839062e-03],
        [ 1.15496498e-02,  1.84064042e-02,  1.07421465e-02, ...,
          5.71913924e-03, -1.75766507e-03,  1.30740916e-02],
        ...,
        [ 2.03288477e-02, -7.98869040e-03,  2.42463928e-02, ...,
         -5.90799609e-03,  9.65785887e-03, -1.26922876e-02],
        [-6.60760188e-03,  3.74193955e-03, -2.76752817e-03, ...,
         -1.00113945e-02, -5.20520052e-03,  2.33142963e-03],
        [ 4.02665930e-03,  1.88150275e-02,  1.21134911e-02, ...,
          8.95798486e-03, -1.12047391e-02,  1.22876707e-02]],

...,

[[-2.30612271e-02, -2.71627568e-02, -2.97727343e-03, ...,
         -1.52611975e-02, -1.08289560e-02,  4.54727188e-03],
        [-5.37802093e-03,  1.53265456e-02, -7.98696093e-03, ...,
          5.77224419e-03,  3.07626091e-02,  6.27357746e-03],
        [ 7.67199555e-03,  1.04494197e-02, -1.23493625e-02, ...,
          4.24701255e-04, -9.98084806e-03,  5.56567824e-03],
        ...,
        [ 1.42724207e-02,  3.21529619e-02, -1.39995655e-02, ...,
          1.84636272e-03,  3.53092357e-04, -5.84522309e-03],
        [ 1.17971171e-02,  9.07169748e-03, -9.52398684e-03, ...,
          1.18852360e-03,  3.43973120e-03, -7.44402781e-03],
        [ 1.10389059e-02, -1.50724845e-02,  4.58929833e-04, ...,
          1.01573355e-02,  1.08590862e-02,  1.08767196e-03]]],      dtype=float32)
                }
            }
        },
        'x_layers_5': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': Array([-0.0082708 ,  0.00023509, -0.00894445, ..., -0.01141324,
        0.00119758, -0.00033616], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.1104382, 1.000599 , 1.0856799, ..., 0.9565084, 1.2328953,
        1.0505954]], dtype=float32),
                        'lora_a': Array([[-0.10848766],
       [ 0.2945134 ],
       [-0.26035208],
       ...,
       [-0.7982494 ],
       [ 0.7376163 ],
       [-0.08403846]], dtype=float32),
                        'lora_b': Array([[ 4.8453803e-06],
       [-2.0351052e-05],
       [-7.0559599e-06],
       ...,
       [-1.9553898e-05],
       [ 0.0000000e+00],
       [ 9.8038208e-06]], dtype=float32),
                        'w': Array([[-0.00602971, -0.02676638,  0.0168558 , ..., -0.00994777,
         0.02569861,  0.02104281],
       [-0.03416809,  0.02381924, -0.04968129, ...,  0.03150865,
        -0.00796047,  0.03368018],
       [-0.00179048, -0.03266168,  0.02378024, ...,  0.01224884,
        -0.03469853, -0.00878385],
       ...,
       [ 0.00312726, -0.00156428,  0.00891732, ..., -0.04592262,
        -0.0226953 , -0.04572573],
       [ 0.00380302, -0.03548504, -0.01470138, ..., -0.03882184,
        -0.0209375 ,  0.02290299],
       [-0.01230227, -0.06000502, -0.02844962, ...,  0.00688603,
        -0.00268077, -0.03837435]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': Array([ 0.00231809,  0.00047749, -0.00265504, ..., -0.00051469,
        0.00263429,  0.00085238], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.215443 , 1.2389   , 1.1621054, ..., 1.2220286, 1.1810489,
        1.1818447]], dtype=float32),
                        'lora_a': Array([[-0.1084674 ],
       [ 0.29451156],
       [-0.2603667 ],
       ...,
       [-0.7982796 ],
       [ 0.7376419 ],
       [-0.08403657]], dtype=float32),
                        'lora_b': Array([[ 1.3122321e-05],
       [-4.3527930e-06],
       [-1.6420618e-05],
       ...,
       [ 1.9002739e-05],
       [ 2.0625587e-06],
       [-1.3179558e-05]], dtype=float32),
                        'w': Array([[ 0.06057307,  0.05345616, -0.02766749, ...,  0.04518164,
        -0.03843925, -0.04467538],
       [ 0.00835851,  0.04028086,  0.03977181, ..., -0.02442977,
         0.0200953 ,  0.02775957],
       [-0.02266712,  0.06793138, -0.04917101, ..., -0.02769876,
        -0.0532935 , -0.02923822],
       ...,
       [ 0.03883331, -0.04628368,  0.01323004, ..., -0.01341316,
         0.04206111,  0.00142196],
       [-0.02760278,  0.02879915,  0.05136937, ..., -0.01124197,
         0.00320576,  0.01919117],
       [-0.03938087, -0.05277129,  0.01173478, ..., -0.00043572,
         0.0437616 , -0.02255625]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': Array([-0.00361701,  0.00014588,  0.00192671, ..., -0.00092808,
        0.00158209,  0.00770174], dtype=float32),
                    'scale': Array([-0.01588804, -0.01103332, -0.01152071, ..., -0.01402099,
        0.01218372,  0.00757895], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': Array([1.0854104, 1.1664495, 1.0141362, ..., 1.0315809, 1.1132778,
       1.0668985], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': Array([[ 2.9983949e-03,  4.5695826e-03, -1.4037583e-03, ...,
         2.0471390e-03, -5.5616634e-04, -4.3982812e-03],
       [ 2.6113386e-03, -3.7855271e-03, -1.1430590e-03, ...,
         1.2536233e-03,  5.8888341e-04,  2.4085233e-03],
       [ 6.1706378e-04, -3.5343072e-03,  3.6754673e-03, ...,
        -1.0473594e-03, -2.6400243e-03, -2.6224724e-03],
       ...,
       [-2.7933377e-03,  9.0840273e-04, -8.4660889e-05, ...,
        -4.1713614e-05,  1.6534668e-03,  2.1746625e-04],
       [-1.4082552e-03, -1.5240485e-03,  8.5433078e-04, ...,
        -1.5572939e-04, -8.7523012e-04, -2.2009872e-03],
       [ 1.7080423e-04, -1.5451813e-03, -1.1473884e-03, ...,
        -3.0459089e-03,  3.4824025e-05,  1.3262454e-03]], dtype=float32),
                    'dora_m': Array([[[0.41909516, 0.46365348, 0.5157949 , ..., 0.47050354,
         0.4395923 , 0.38403347],
        [0.52106434, 0.35620415, 0.4665356 , ..., 0.63351905,
         0.34586868, 0.34701923],
        [0.45828435, 0.42735407, 0.45630267, ..., 0.48785067,
         0.37983793, 0.39578223],
        ...,
        [0.486351  , 0.4917389 , 0.45206386, ..., 0.4489878 ,
         0.4665377 , 0.40077618],
        [0.61392134, 0.4880771 , 0.46362522, ..., 0.43855503,
         0.41498172, 0.5353842 ],
        [0.5709485 , 0.42146602, 0.47688207, ..., 0.5632734 ,
         0.520023  , 0.48006696]]], dtype=float32),
                    'lora_a': Array([[-0.10848039],
       [ 0.29453456],
       [-0.2603809 ],
       ...,
       [-0.798249  ],
       [ 0.73761684],
       [-0.08400826]], dtype=float32),
                    'lora_b': Array([[ 1.0496294e-05],
       [ 5.7776816e-08],
       [ 4.2609404e-06],
       ...,
       [ 6.3800137e-08],
       [-8.2622355e-06],
       [-1.0247703e-05]], dtype=float32),
                    'w': Array([[[-7.63362879e-03,  5.10084257e-03, -2.08918564e-02, ...,
         -1.14847347e-02,  1.26297809e-02, -2.70505290e-04],
        [ 1.42549560e-03, -2.30036471e-02, -1.78317297e-02, ...,
         -3.46740684e-03, -9.71160643e-03,  1.18957236e-02],
        [ 1.04235457e-02, -1.16101396e-03, -1.41830705e-02, ...,
          1.42017473e-02, -9.16350726e-03, -7.10327178e-03],
        ...,
        [-1.11079877e-02, -2.10176408e-02, -7.89483637e-03, ...,
         -3.19734146e-03,  1.45656744e-03,  8.61223228e-03],
        [-1.26707973e-02,  9.41790640e-03, -1.63667207e-03, ...,
         -3.07305576e-03,  3.62688303e-03,  1.33423358e-02],
        [-2.00318801e-03, -1.57821178e-02,  1.17536416e-04, ...,
         -3.37612652e-03, -1.36024542e-02,  2.33557844e-03]],

...,

[[ 1.29230488e-02, -1.24336779e-02,  1.27518242e-02, ...,
         -1.73731763e-02, -2.01447140e-02, -2.70947861e-03],
        [-1.85279734e-02,  4.07904806e-03, -2.91116349e-03, ...,
          3.54991818e-04, -1.13391196e-02,  1.60872890e-03],
        [ 6.38186233e-03,  6.76335953e-03,  1.04324734e-02, ...,
          1.50489118e-02, -5.08294720e-03, -6.67493325e-03],
        ...,
        [ 2.76850406e-02,  3.05225533e-02,  1.50380582e-02, ...,
          1.40753342e-02, -1.41735123e-02,  2.10294109e-02],
        [-3.27004190e-03,  6.93722046e-04, -1.51707726e-02, ...,
          3.33688594e-02, -3.51021794e-04,  2.66710799e-02],
        [-9.72767826e-03, -3.45314061e-03, -8.60371813e-03, ...,
         -3.05569042e-02,  2.07188912e-02, -1.41795063e-02]]],      dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': Array([0.13068558, 0.11264031, 0.11708908, 0.12530498, 0.12238726,
       0.13479206, 0.0961272 , 0.10192457, 0.06898148, 0.11135861,
       0.11440782, 0.11942353, 0.11138015, 0.11427903, 0.15602514,
       0.10358352, 0.12103997, 0.10196049, 0.11657108, 0.1105193 ,
       0.13546531, 0.10212965, 0.12147649, 0.11892303, 0.14143457,
       0.13192987, 0.12624545, 0.12940401, 0.11032961, 0.11966604,
       0.1251902 , 0.12702945, 0.12194424, 0.12021966, 0.10040382,
       0.09689584, 0.13449894, 0.145295  , 0.12237985, 0.1316158 ,
       0.14289606, 0.1149406 , 0.12861389, 0.09718668, 0.10809228,
       0.11319933, 0.1056805 , 0.11801772, 0.13401496, 0.13912912,
       0.1070198 , 0.10733105, 0.12350067, 0.12399949, 0.12067261,
       0.09138734, 0.1365333 , 0.1002665 , 0.13340193, 0.12104555,
       0.11990233, 0.10155478, 0.1109368 , 0.12974575, 0.11362689,
       0.13002796, 0.13385442, 0.1020591 , 0.10243788, 0.12618516,
       0.13293782, 0.10212671, 0.0935626 , 0.12699246, 0.14163223,
       0.1277229 , 0.11770076, 0.14909612, 0.10548238, 0.09269526],      dtype=float32)
                },
                'post': {
                    'b': Array([ 0.00065614,  0.00162599,  0.00121886, ..., -0.00042695,
        0.00442083,  0.00376454], dtype=float32),
                    'dora_m': Array([[[0.3576581 , 0.34659395, 0.32720327, ..., 0.3338639 ,
         0.3732263 , 0.3346922 ],
        [0.35202202, 0.34896043, 0.3354163 , ..., 0.35194144,
         0.3715785 , 0.35806853],
        [0.34772304, 0.3472168 , 0.34599262, ..., 0.357898  ,
         0.3603679 , 0.35333833],
        ...,
        [0.3814033 , 0.40708306, 0.36061695, ..., 0.39204147,
         0.3613276 , 0.36538827],
        [0.36458465, 0.35296842, 0.3493063 , ..., 0.36191922,
         0.3541788 , 0.36208493],
        [0.35711572, 0.37709045, 0.37916556, ..., 0.36530858,
         0.35702354, 0.36383477]]], dtype=float32),
                    'lora_a': Array([[-0.10849015],
       [ 0.2945515 ],
       [-0.2603605 ],
       ...,
       [-0.7982543 ],
       [ 0.73762953],
       [-0.08401051]], dtype=float32),
                    'lora_b': Array([[ 3.5738333e-06],
       [ 7.8272233e-06],
       [ 1.9898751e-05],
       ...,
       [-1.5525065e-05],
       [ 2.0402227e-05],
       [ 6.9044859e-07]], dtype=float32),
                    'w': Array([[[-2.49264226e-03,  1.17608486e-02,  1.48023441e-02, ...,
         -1.27285868e-02,  4.40729549e-03,  1.67888775e-02],
        [ 1.92789957e-02, -4.13750857e-03, -1.37377379e-03, ...,
         -8.48557975e-04, -3.75018991e-03,  1.16395194e-03],
        [-4.72128903e-03,  2.33182032e-02,  3.60964076e-03, ...,
         -2.32205801e-02,  4.94472077e-03,  1.40123470e-02],
        ...,
        [ 1.15733771e-02,  5.74059924e-03, -3.51108727e-03, ...,
          3.92256508e-04, -5.45613840e-03, -6.04295032e-03],
        [ 1.52369856e-03,  1.14200599e-02,  1.37395766e-02, ...,
         -2.21631420e-03,  1.10637462e-02,  1.66783351e-02],
        [ 2.96310778e-03,  1.01144752e-02,  1.01158349e-02, ...,
         -3.47329327e-03, -1.06788361e-02, -9.48280375e-03]],

...,

[[ 5.33286156e-03,  4.65261610e-03, -4.57288744e-03, ...,
         -1.17293769e-03,  1.02246245e-02,  3.17146722e-03],
        [ 1.14073955e-04, -8.93527269e-03,  1.22774597e-02, ...,
         -1.29055474e-02,  1.76185332e-02, -5.55372750e-03],
        [ 8.42548721e-03,  7.63535034e-03,  5.08680940e-03, ...,
          1.00998925e-02,  5.60012460e-03,  3.20164138e-03],
        ...,
        [-1.29430825e-02, -1.70630645e-02,  6.33569725e-04, ...,
         -2.31857924e-03, -4.24584793e-03, -1.89849064e-02],
        [-1.04850493e-02,  8.94758850e-03,  3.43736378e-03, ...,
         -9.55532677e-03, -5.64501993e-03,  1.00068869e-02],
        [ 1.50155732e-02,  6.59107976e-03,  2.04607006e-02, ...,
         -6.30389946e-03,  1.74301548e-03,  6.87498460e-03]]],      dtype=float32)
                },
                'query': {
                    'b': Array([[-0.00535969,  0.00544131,  0.00895877, ..., -0.01104905,
        -0.00018965, -0.00259527],
       [-0.00518589,  0.0229747 ,  0.03777637, ...,  0.00089205,
         0.01973232, -0.03037073],
       [ 0.00032148,  0.01091193, -0.01324205, ..., -0.00556419,
         0.01459017,  0.01027916],
       ...,
       [-0.02908402, -0.00352269, -0.01038569, ..., -0.02177537,
         0.02019865, -0.00165102],
       [-0.00203704, -0.00681012,  0.00786141, ..., -0.01706576,
         0.01285954, -0.00309793],
       [-0.00697265, -0.01840293, -0.01635662, ...,  0.00518259,
        -0.01693589,  0.00704248]], dtype=float32),
                    'dora_m': Array([[[0.5036897 , 0.5315111 , 0.59064907, ..., 0.5929797 ,
         0.5143985 , 0.5286374 ],
        [0.55752796, 0.45457363, 0.58689094, ..., 0.6105448 ,
         0.52719086, 0.47412613],
        [0.5719777 , 0.5563024 , 0.5786502 , ..., 0.5948315 ,
         0.49188778, 0.5158359 ],
        ...,
        [0.77866226, 0.716198  , 0.53931004, ..., 0.67755073,
         0.6170952 , 0.6350767 ],
        [0.4735056 , 0.5721415 , 0.6613448 , ..., 0.6435375 ,
         0.6517233 , 0.51951075],
        [0.6562359 , 0.81167537, 0.8000877 , ..., 0.7500054 ,
         0.6153127 , 0.77464855]]], dtype=float32),
                    'lora_a': Array([[-0.10847327],
       [ 0.2945178 ],
       [-0.26036623],
       ...,
       [-0.79827034],
       [ 0.7376349 ],
       [-0.08403327]], dtype=float32),
                    'lora_b': Array([[-2.2262445e-07],
       [ 2.0707103e-05],
       [-1.7573560e-05],
       ...,
       [ 9.7449993e-06],
       [-1.6595346e-05],
       [-6.6385437e-06]], dtype=float32),
                    'w': Array([[[-3.59171070e-03,  4.86029225e-04,  1.73513964e-02, ...,
         -2.03544274e-02, -7.29639363e-03, -1.37512004e-02],
        [ 3.51476576e-03, -4.14279569e-03,  5.83695527e-03, ...,
          4.22853889e-04,  1.60983242e-02,  1.52249995e-03],
        [ 1.19537720e-03,  8.63350928e-03, -1.61050875e-02, ...,
          1.76351350e-02, -1.06409669e-03, -1.31810606e-02],
        ...,
        [ 5.92035335e-03, -5.20166615e-03, -9.49679222e-03, ...,
          2.86339456e-03,  8.24203063e-03,  7.12555787e-03],
        [-1.71734393e-02, -2.81062094e-04, -9.40955617e-03, ...,
         -1.18391523e-02,  3.19176801e-02,  2.29466725e-02],
        [-1.70886412e-03,  9.49707627e-03, -2.41508223e-02, ...,
          3.97554711e-02, -1.66079821e-03, -4.81888931e-03]],

...,

[[-1.08893197e-02,  2.01132782e-02, -1.41311232e-02, ...,
          4.49838080e-02,  1.88883208e-02, -2.49197595e-02],
        [ 1.31158363e-02, -1.01136826e-02, -1.65932309e-02, ...,
          2.25598309e-02,  1.38090095e-02,  1.44756946e-03],
        [-2.09184662e-02, -4.16760426e-03, -1.00047747e-02, ...,
         -2.33833566e-02, -2.47874651e-02, -1.74675826e-02],
        ...,
        [ 1.01681864e-02,  2.75148526e-02, -3.07024946e-03, ...,
         -1.39357764e-02, -2.08011828e-02,  2.21577492e-02],
        [-3.29795070e-02,  7.55552202e-04, -3.07305008e-02, ...,
         -6.28283992e-03, -1.76763702e-02,  9.68733989e-03],
        [-2.69406033e-03,  4.98209661e-03,  4.78894822e-03, ...,
         -4.48810458e-02, -1.16679259e-02, -1.80438738e-02]]],      dtype=float32)
                },
                'value': {
                    'b': Array([[ 1.7673090e-04, -2.0610308e-03, -1.2899771e-03, ...,
         1.7491290e-03, -1.8086215e-04, -4.2822282e-04],
       [-1.4554589e-03, -2.9118320e-03,  3.2678701e-05, ...,
         2.7426528e-03,  1.5872034e-04, -2.5911438e-03],
       [ 2.2816991e-04, -3.2166156e-03, -7.0322072e-04, ...,
        -1.3632405e-03,  1.9719843e-03, -6.0191058e-04],
       ...,
       [ 2.8800408e-03,  1.0955781e-03, -1.1744021e-03, ...,
         2.8533933e-03, -4.6616411e-03, -2.5245901e-03],
       [ 3.6332270e-03, -3.3154138e-03, -9.8116521e-04, ...,
        -7.3912339e-03, -1.3474902e-03, -3.7023891e-03],
       [ 1.6156649e-03, -1.6597542e-03,  1.2324060e-03, ...,
        -3.4989812e-04, -7.1168086e-04, -4.4309464e-04]], dtype=float32),
                    'dora_m': Array([[[0.41400144, 0.38582146, 0.3692162 , ..., 0.38814723,
         0.4382676 , 0.37254483],
        [0.43203825, 0.43004572, 0.39703834, ..., 0.42876756,
         0.4383296 , 0.4298327 ],
        [0.42698845, 0.40252104, 0.4300438 , ..., 0.42077082,
         0.44830185, 0.41573045],
        ...,
        [0.45372477, 0.47391203, 0.43469742, ..., 0.48358446,
         0.42743257, 0.46934012],
        [0.43467194, 0.42674547, 0.42415634, ..., 0.42670807,
         0.423486  , 0.44669628],
        [0.42011318, 0.4309268 , 0.44713402, ..., 0.41064027,
         0.42260537, 0.41634086]]], dtype=float32),
                    'lora_a': Array([[-0.10848958],
       [ 0.29451337],
       [-0.26035875],
       ...,
       [-0.7982738 ],
       [ 0.73762494],
       [-0.08403989]], dtype=float32),
                    'lora_b': Array([[-2.0747808e-05],
       [-1.8799799e-05],
       [ 1.5276692e-05],
       ...,
       [ 9.7985057e-06],
       [-1.5863603e-05],
       [ 4.9019341e-06]], dtype=float32),
                    'w': Array([[[-1.94498431e-02, -7.64790457e-03, -6.29008841e-03, ...,
         -5.54918498e-03,  2.08781715e-02, -8.77577718e-03],
        [ 1.24037508e-02, -7.79988663e-03,  1.69514921e-02, ...,
          1.23843411e-02,  1.49343666e-02, -8.53807293e-03],
        [ 1.97821800e-02, -4.14963113e-03,  1.94442533e-02, ...,
          1.06186885e-02,  2.99014500e-03, -1.60769522e-02],
        ...,
        [-7.28181889e-03, -2.31254287e-02,  1.61775816e-02, ...,
          5.33160521e-03,  6.89735729e-03,  3.29807326e-02],
        [ 6.98302267e-03,  4.92673740e-03, -1.29372906e-02, ...,
          1.28492406e-02, -1.33066913e-02,  1.97734907e-02],
        [ 6.50113123e-03,  5.44579467e-03, -1.27206072e-02, ...,
          2.45580599e-02, -2.19962932e-02, -1.54230632e-02]],

...,

[[-1.30176395e-02,  1.15965446e-02, -1.36257289e-03, ...,
          1.13581447e-02,  8.71351361e-03, -6.23817975e-03],
        [ 5.81024494e-03,  1.62437931e-02, -5.57216350e-04, ...,
         -2.28420808e-03,  2.54431181e-03,  9.41395701e-04],
        [ 2.17519812e-02, -1.07939215e-02, -8.15818086e-03, ...,
         -1.20645314e-02,  1.17858164e-02,  2.23123282e-02],
        ...,
        [-3.23988474e-03,  4.68354439e-03, -8.89800303e-03, ...,
         -1.05521753e-02,  6.07082248e-03,  7.68872537e-03],
        [-6.99350843e-03,  2.20081629e-03, -1.68537572e-02, ...,
          3.71227157e-03, -5.02140680e-03,  2.17879447e-03],
        [-8.09752475e-03, -3.74472264e-04,  1.37547473e-03, ...,
         -9.78743657e-03,  1.00383982e-02,  8.58860463e-03]]],      dtype=float32)
                }
            }
        },
        'x_layers_6': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': Array([-0.05319161, -0.00180769, -0.00665944, ..., -0.00591564,
       -0.01293479, -0.01065589], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[2.1763372, 1.0366162, 1.0428661, ..., 1.0158364, 1.0948471,
        0.989442 ]], dtype=float32),
                        'lora_a': Array([[-0.10848856],
       [ 0.29455298],
       [-0.2603507 ],
       ...,
       [-0.7982768 ],
       [ 0.7376256 ],
       [-0.08403549]], dtype=float32),
                        'lora_b': Array([[ 0.0000000e+00],
       [ 1.3541087e-05],
       [-1.7961926e-05],
       ...,
       [ 4.5640859e-06],
       [-5.0116182e-06],
       [-1.5983172e-05]], dtype=float32),
                        'w': Array([[-0.13107388, -0.00877222, -0.01354472, ...,  0.00660291,
         0.00827086,  0.06084311],
       [-0.08909545, -0.00488692, -0.04813811, ..., -0.01833421,
        -0.02547176, -0.04040889],
       [-0.03659885, -0.00826377, -0.00942399, ..., -0.01570849,
        -0.0093856 , -0.01922558],
       ...,
       [-0.0066282 , -0.03163596, -0.01922997, ...,  0.00631564,
        -0.0245108 , -0.03882673],
       [-0.01133524, -0.01882924, -0.01039014, ..., -0.02043537,
         0.06204742,  0.03220206],
       [-0.05412352,  0.02532915,  0.02422992, ..., -0.03567279,
        -0.02308617,  0.01022565]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': Array([ 0.00293988,  0.0009932 ,  0.00107772, ..., -0.00040506,
        0.0001442 ,  0.00212154], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.582064 , 1.3840272, 1.8315448, ..., 1.5262436, 1.5674193,
        1.4928324]], dtype=float32),
                        'lora_a': Array([[-0.10846734],
       [ 0.29454488],
       [-0.26035565],
       ...,
       [-0.7982427 ],
       [ 0.7376237 ],
       [-0.08399893]], dtype=float32),
                        'lora_b': Array([[7.8029088e-06],
       [1.2485796e-05],
       [1.3785990e-05],
       ...,
       [8.5935590e-06],
       [7.7040713e-06],
       [4.2492179e-06]], dtype=float32),
                        'w': Array([[-0.4611281 , -0.20447981, -0.2977535 , ...,  0.23038045,
         0.46599442, -0.1837977 ],
       [ 0.0019854 ,  0.006754  , -0.01749402, ..., -0.05501841,
         0.04297751,  0.01049735],
       [ 0.02726045,  0.03819789,  0.0172031 , ..., -0.00300778,
        -0.02944737,  0.00064869],
       ...,
       [-0.06164307, -0.00551394, -0.02048076, ..., -0.00060017,
        -0.01957849, -0.02914051],
       [-0.02144488,  0.02477437, -0.01749869, ...,  0.00511124,
         0.00741991,  0.02064732],
       [-0.02758567,  0.0044741 , -0.01848485, ..., -0.02339551,
         0.01778207, -0.0161617 ]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': Array([ 0.0018998 ,  0.00821097, -0.00399487, ..., -0.00671882,
        0.00488133,  0.00149973], dtype=float32),
                    'scale': Array([-0.01244817,  0.00892163, -0.01021829, ..., -0.02017842,
       -0.01239914,  0.01176315], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': Array([1.1080961, 1.1210177, 1.0306809, ..., 1.050785 , 1.0788523,
       1.0700463], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': Array([[-3.1537301e-04, -1.3355116e-03, -6.2528707e-04, ...,
        -1.7741041e-03,  2.3211220e-04, -1.1813914e-03],
       [-1.0428097e-03, -3.3338604e-04,  1.5919108e-03, ...,
        -1.4696107e-03,  1.1327536e-03,  2.0284744e-03],
       [ 2.1186552e-03, -1.5467203e-04, -5.1601962e-03, ...,
        -1.9826847e-03,  1.6850738e-04, -4.3730572e-04],
       ...,
       [-8.5889758e-04, -4.9279694e-04, -4.7004360e-04, ...,
         8.0387533e-04, -8.7275222e-04,  1.2364115e-03],
       [-1.6918234e-03, -1.5437314e-03,  1.1954332e-03, ...,
        -7.0258800e-05,  2.0370714e-03,  1.2066830e-03],
       [ 1.9361990e-03,  2.2996487e-03, -1.3164624e-04, ...,
        -4.9002131e-04, -5.7391531e-04,  1.3540803e-03]], dtype=float32),
                    'dora_m': Array([[[0.52373266, 0.5184106 , 0.4956361 , ..., 0.5128289 ,
         0.42290187, 0.42756844],
        [0.50768423, 0.47346702, 0.43998018, ..., 0.50297916,
         0.45265538, 0.43752894],
        [0.47698972, 0.49412185, 0.52190673, ..., 0.45810694,
         0.5229571 , 0.603542  ],
        ...,
        [0.39922553, 0.51330996, 0.46458286, ..., 0.43113863,
         0.44072387, 0.46399748],
        [0.47514218, 0.35729942, 0.46515533, ..., 0.3387646 ,
         0.36569226, 0.49622506],
        [0.41772434, 0.40683544, 0.54288346, ..., 0.43048555,
         0.455812  , 0.39899635]]], dtype=float32),
                    'lora_a': Array([[-0.10848065],
       [ 0.2945496 ],
       [-0.26035973],
       ...,
       [-0.798245  ],
       [ 0.73761743],
       [-0.08400619]], dtype=float32),
                    'lora_b': Array([[ 1.0726671e-05],
       [-5.2051796e-06],
       [-6.7274395e-06],
       ...,
       [-3.9780210e-07],
       [ 2.6168518e-06],
       [ 3.6164049e-06]], dtype=float32),
                    'w': Array([[[ 2.24064197e-02, -5.44897374e-03, -7.02082319e-03, ...,
         -1.11840591e-02,  5.14022214e-03,  4.75016190e-03],
        [ 2.71372520e-03, -1.44458525e-02, -1.66786220e-02, ...,
          2.48380173e-02,  5.30734845e-03, -4.04858962e-03],
        [-2.02985518e-02, -2.22432688e-02,  5.04163140e-03, ...,
          1.05001461e-02, -5.45279123e-03,  1.94659010e-02],
        ...,
        [-5.60708297e-03,  3.33053153e-03,  5.22669929e-04, ...,
          8.10892379e-05,  1.56823394e-03,  8.21985491e-03],
        [-3.26173590e-03, -9.37842578e-03, -2.09067836e-02, ...,
          3.57581908e-03,  1.43249426e-02,  2.75801085e-02],
        [-9.59258806e-03, -6.47648470e-03,  3.48970410e-03, ...,
          7.60874245e-03,  1.22080911e-02, -2.02672509e-03]],

...,

[[-4.28056484e-03, -1.93030061e-03,  4.81113559e-03, ...,
          4.17014398e-03, -7.71327922e-03, -1.40038319e-02],
        [ 1.41333453e-02, -8.93208664e-03, -8.02380871e-03, ...,
          7.98764359e-03,  5.41705173e-03,  1.27402926e-02],
        [-1.76554769e-02, -2.69824755e-03,  1.54527044e-02, ...,
          1.34667978e-02,  6.39165938e-03,  9.21813771e-03],
        ...,
        [ 2.51757596e-02,  4.08009719e-03, -1.52747296e-02, ...,
         -2.46939585e-02,  1.07817482e-02, -2.32611988e-02],
        [-2.06047632e-02,  9.03889630e-03, -1.09267971e-02, ...,
          3.04490415e-04, -3.37161683e-03,  3.82427089e-02],
        [ 4.00011020e-04,  2.06485973e-03, -9.59498808e-03, ...,
         -6.34575868e-03,  2.44500060e-02,  1.71611514e-02]]],      dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': Array([0.11642151, 0.08755226, 0.13129562, 0.11135118, 0.1541929 ,
       0.14946258, 0.16739756, 0.07969241, 0.07921032, 0.10419153,
       0.1310537 , 0.09849536, 0.12754586, 0.12130971, 0.16390255,
       0.12960353, 0.11012015, 0.12605913, 0.08225828, 0.12483724,
       0.07553666, 0.10880233, 0.16587763, 0.15890025, 0.17667218,
       0.12203955, 0.16270383, 0.13956173, 0.17609157, 0.1329435 ,
       0.17093259, 0.1487696 , 0.13642834, 0.12862892, 0.11986644,
       0.14018399, 0.13569863, 0.12352876, 0.1390964 , 0.13789728,
       0.11127125, 0.1740751 , 0.11412632, 0.15834849, 0.1430574 ,
       0.19045705, 0.14197344, 0.15991192, 0.1673336 , 0.08320697,
       0.11853421, 0.13975696, 0.10554477, 0.07266311, 0.1530208 ,
       0.14626922, 0.1599174 , 0.10922667, 0.15007749, 0.10078926,
       0.11527258, 0.09056383, 0.10290945, 0.11431269, 0.10214151,
       0.11281752, 0.14514863, 0.09825553, 0.12906459, 0.09839334,
       0.1310849 , 0.17151259, 0.08175332, 0.13591628, 0.17111905,
       0.07780149, 0.14896274, 0.15863752, 0.11145978, 0.16233292],      dtype=float32)
                },
                'post': {
                    'b': Array([ 0.00238558,  0.00539059,  0.00254566, ..., -0.00227536,
        0.00215697,  0.0025347 ], dtype=float32),
                    'dora_m': Array([[[0.40734407, 0.4142747 , 0.38511306, ..., 0.3973461 ,
         0.3992566 , 0.3733829 ],
        [0.37583554, 0.37830633, 0.35837826, ..., 0.36783397,
         0.37277442, 0.38693306],
        [0.40044218, 0.3881213 , 0.35556915, ..., 0.3340691 ,
         0.36439848, 0.3595313 ],
        ...,
        [0.40285832, 0.38156903, 0.3725513 , ..., 0.3754986 ,
         0.39065114, 0.3773722 ],
        [0.34197873, 0.31930634, 0.35415754, ..., 0.37947983,
         0.32948345, 0.29135296],
        [0.35525674, 0.39039448, 0.3623697 , ..., 0.35172096,
         0.38713056, 0.337886  ]]], dtype=float32),
                    'lora_a': Array([[-0.10849077],
       [ 0.29455334],
       [-0.26037475],
       ...,
       [-0.7982489 ],
       [ 0.7376283 ],
       [-0.08401097]], dtype=float32),
                    'lora_b': Array([[-3.5967432e-06],
       [ 1.2724161e-05],
       [-1.7620316e-05],
       ...,
       [ 1.7458213e-05],
       [-4.2377733e-06],
       [ 1.2058999e-05]], dtype=float32),
                    'w': Array([[[-1.31261721e-02, -2.97298040e-02,  8.55484325e-03, ...,
          2.99458811e-03,  7.99451582e-03,  4.69723949e-03],
        [ 2.37402972e-02, -5.83681231e-03, -1.17273564e-02, ...,
         -4.31737164e-03, -1.61416568e-02,  5.95104136e-03],
        [-8.93504638e-03,  1.82599935e-03,  8.56268592e-03, ...,
         -1.26973065e-02, -5.25536062e-03, -6.47541368e-03],
        ...,
        [ 2.20728423e-02, -1.01757552e-02, -2.52463436e-03, ...,
         -8.62695649e-03,  1.56933945e-02, -1.77065507e-02],
        [ 4.22290387e-03, -1.19603090e-02,  8.87258165e-03, ...,
          2.84907804e-03, -1.67047736e-04,  3.99949402e-03],
        [ 6.23347703e-03,  2.00265143e-02, -9.35546495e-03, ...,
          1.93768144e-02, -1.65404938e-02,  5.31815866e-04]],

...,

[[-1.76674947e-02, -3.76421376e-03,  1.47921732e-03, ...,
          1.29456343e-02,  4.27357294e-03,  5.15852123e-03],
        [ 2.25980859e-03,  1.39413550e-02,  7.51878368e-04, ...,
          4.16671112e-03,  2.33600629e-04, -4.58875642e-04],
        [ 2.03958916e-06, -2.20134063e-03, -8.97115748e-03, ...,
         -1.51937432e-03,  1.53056076e-02,  4.09533083e-03],
        ...,
        [-2.85116630e-03,  2.10857601e-03,  5.37850382e-03, ...,
         -3.66166560e-03, -1.59080066e-02,  4.00653947e-03],
        [ 1.76424384e-02, -8.02353863e-03,  1.91531260e-03, ...,
          1.64852082e-03,  2.21708702e-04,  2.77315639e-03],
        [-5.73996827e-03,  1.43148312e-02,  4.94344532e-03, ...,
         -1.78103801e-02, -2.68698321e-03, -1.45744148e-03]]],      dtype=float32)
                },
                'query': {
                    'b': Array([[ 6.9429409e-03, -4.2634704e-03, -2.7791863e-02, ...,
        -1.0255985e-02, -1.0615559e-05, -2.6138142e-02],
       [-1.2106268e-03, -1.1853982e-02, -3.1291507e-02, ...,
         3.1470947e-02, -4.6250102e-04, -3.4001581e-02],
       [-5.2930247e-02, -2.2960834e-02, -2.2257559e-02, ...,
         5.8740806e-02, -3.5103314e-02,  4.9581606e-02],
       ...,
       [-6.5015107e-03,  1.1675995e-02,  8.1795076e-04, ...,
         1.6125415e-03,  5.1611493e-04,  1.4375899e-02],
       [ 9.2227589e-03, -9.7014355e-03,  1.6348656e-02, ...,
        -3.0453945e-02,  7.7625657e-03, -1.3052466e-02],
       [-2.1310246e-03, -1.3935447e-02, -7.7975774e-04, ...,
         1.2236374e-02, -2.9945837e-03, -3.2102251e-03]], dtype=float32),
                    'dora_m': Array([[[0.65667266, 0.6663924 , 0.77261156, ..., 0.6853928 ,
         0.6355011 , 0.92344654],
        [0.66794187, 0.6447533 , 0.53270817, ..., 0.9763482 ,
         0.62813175, 0.6034127 ],
        [0.48975408, 0.5588568 , 0.7657893 , ..., 0.7363243 ,
         0.5852501 , 0.52805364],
        ...,
        [0.78401047, 0.577257  , 0.77208203, ..., 0.78680265,
         0.7222442 , 0.64824706],
        [0.53842753, 0.5179846 , 0.6764868 , ..., 0.4778051 ,
         0.5332382 , 0.6958381 ],
        [0.56977093, 0.5771867 , 0.45513096, ..., 0.6232433 ,
         0.61543715, 0.58392256]]], dtype=float32),
                    'lora_a': Array([[-0.10848413],
       [ 0.29451963],
       [-0.26036507],
       ...,
       [-0.79827285],
       [ 0.7376271 ],
       [-0.08402946]], dtype=float32),
                    'lora_b': Array([[ 1.8218336e-05],
       [-2.0272496e-06],
       [-2.7939516e-06],
       ...,
       [-1.8168877e-05],
       [-2.1071673e-05],
       [ 1.2344961e-05]], dtype=float32),
                    'w': Array([[[-3.16672074e-03, -4.12859917e-02,  3.21716592e-02, ...,
          2.45895912e-03,  8.08126386e-03, -9.22187325e-03],
        [-6.76687434e-03, -2.07112320e-02,  5.13712093e-02, ...,
          7.17079872e-03, -2.47327350e-02,  1.19166179e-02],
        [ 2.89679803e-02,  2.35539190e-02, -7.39458501e-02, ...,
          1.19256554e-02,  2.08058786e-02, -3.49808764e-03],
        ...,
        [ 2.40502618e-02, -2.07911748e-02,  8.93004518e-03, ...,
         -2.51299459e-02,  2.76682843e-02,  4.03891085e-03],
        [ 2.68634390e-02,  3.79768834e-02,  2.77351253e-02, ...,
         -2.39275396e-03, -4.56889421e-02, -2.58478243e-02],
        [ 1.85000955e-03, -1.42436158e-02, -9.64279659e-03, ...,
          1.45759247e-02,  7.56929256e-03, -2.44231988e-02]],

...,

[[-6.21598586e-03,  8.93101934e-03, -1.86101720e-02, ...,
          1.13250483e-02,  1.06790354e-02, -2.76543386e-02],
        [ 1.37867238e-02, -8.71839374e-03, -3.37770469e-02, ...,
         -1.38885714e-02, -1.04870023e-02, -1.42791821e-02],
        [-8.93813092e-03,  2.45561004e-02,  6.13623625e-03, ...,
          4.75202966e-03,  1.26813389e-02,  4.04227711e-03],
        ...,
        [ 2.35408582e-02, -8.72598030e-03, -1.76732186e-02, ...,
          2.81246589e-03,  2.43727863e-02, -1.42596839e-02],
        [-2.70808507e-02, -2.46383362e-02, -2.90598087e-02, ...,
         -6.43516891e-03,  3.99530344e-02,  3.38392779e-02],
        [-2.20168545e-03,  1.46329468e-02, -7.26030860e-03, ...,
         -2.56383326e-03, -5.55172889e-03,  1.80497952e-02]]],      dtype=float32)
                },
                'value': {
                    'b': Array([[-1.0620573e-03, -8.5262273e-04, -2.3737380e-03, ...,
         1.3075229e-03, -1.5832923e-03, -4.5124182e-04],
       [-1.0804366e-03,  8.5270044e-04,  2.5770220e-03, ...,
        -2.9048433e-03, -3.7585336e-04, -4.6495227e-03],
       [ 2.8058852e-03,  2.1279312e-03,  1.8310938e-03, ...,
         3.3984135e-03, -8.4306856e-05,  1.5666549e-03],
       ...,
       [ 1.6452165e-03,  3.6395458e-04, -8.0337049e-04, ...,
         2.2301511e-04,  3.2636215e-04, -7.0281944e-04],
       [ 6.3909398e-04,  1.7140268e-03, -7.0529274e-04, ...,
        -1.2073840e-03,  1.0770707e-03, -1.3171440e-03],
       [-6.2980293e-04, -1.3103519e-03,  3.3602221e-03, ...,
         3.4310108e-03, -3.3526905e-03,  3.5224902e-04]], dtype=float32),
                    'dora_m': Array([[[0.461893  , 0.4852995 , 0.4407823 , ..., 0.46061954,
         0.44132587, 0.42631418],
        [0.44629288, 0.45398617, 0.42002195, ..., 0.42762095,
         0.45111123, 0.43131846],
        [0.44682208, 0.43617755, 0.42100072, ..., 0.37800524,
         0.42348707, 0.39948544],
        ...,
        [0.43719304, 0.43011117, 0.42688945, ..., 0.42733556,
         0.45578548, 0.4310076 ],
        [0.4124297 , 0.3543574 , 0.4106011 , ..., 0.4752097 ,
         0.39857405, 0.33478937],
        [0.40375558, 0.44376794, 0.412278  , ..., 0.4075522 ,
         0.4554359 , 0.36832604]]], dtype=float32),
                    'lora_a': Array([[-0.10849083],
       [ 0.29451343],
       [-0.26036695],
       ...,
       [-0.7982775 ],
       [ 0.7376147 ],
       [-0.08404026]], dtype=float32),
                    'lora_b': Array([[ 1.6810827e-05],
       [ 1.0157380e-05],
       [-1.5686592e-05],
       ...,
       [-8.8132692e-06],
       [-1.3978297e-05],
       [-2.0831447e-05]], dtype=float32),
                    'w': Array([[[-0.00289195,  0.00459336, -0.00052058, ..., -0.02344152,
         -0.01150526, -0.00118089],
        [-0.02425805,  0.01403235,  0.0003381 , ..., -0.0124934 ,
          0.04205531, -0.02273614],
        [-0.00022628,  0.00789206, -0.01157164, ..., -0.00642179,
          0.00537121,  0.00377361],
        ...,
        [-0.0134025 , -0.02511169,  0.02299355, ..., -0.00510895,
         -0.0051118 ,  0.01501753],
        [ 0.01340156,  0.00708515, -0.00953312, ..., -0.00929322,
         -0.00020737, -0.00195217],
        [-0.00274176, -0.01731715,  0.01001492, ..., -0.00381206,
          0.03188181,  0.00717415]],

...,

[[ 0.00171981,  0.0112906 , -0.02009648, ..., -0.00829016,
          0.00360181, -0.00637888],
        [ 0.01464567, -0.020429  , -0.0022388 , ...,  0.00657347,
         -0.01722888,  0.00407795],
        [-0.01575869, -0.00118728, -0.00671169, ...,  0.00680375,
          0.00524415,  0.01559517],
        ...,
        [-0.00369396, -0.00475997, -0.00338401, ..., -0.00040036,
          0.00523991, -0.00302144],
        [ 0.00544856, -0.00823792, -0.02051219, ...,  0.01267082,
         -0.01351453, -0.00835027],
        [ 0.00199931,  0.00934889,  0.00237274, ...,  0.00907233,
         -0.00732404, -0.00648139]]], dtype=float32)
                }
            }
        },
        'x_layers_7': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': Array([-0.00571089,  0.00145912, -0.00137052, ..., -0.00824607,
       -0.00131992, -0.00115036], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[0.9569717, 1.1264422, 0.9940966, ..., 0.9830778, 1.0339265,
        1.0909208]], dtype=float32),
                        'lora_a': Array([[-0.10848985],
       [ 0.29454422],
       [-0.26035282],
       ...,
       [-0.7982782 ],
       [ 0.7376152 ],
       [-0.08403158]], dtype=float32),
                        'lora_b': Array([[ 2.1137868e-05],
       [-8.5558868e-06],
       [ 1.3845156e-06],
       ...,
       [ 1.8250395e-05],
       [ 1.9418456e-05],
       [ 7.2449179e-06]], dtype=float32),
                        'w': Array([[-0.02234673, -0.02884459,  0.0098833 , ..., -0.02191785,
        -0.00469763, -0.04039831],
       [-0.03696799, -0.04514505,  0.01396954, ..., -0.01213033,
        -0.00943626,  0.04773937],
       [ 0.01503859, -0.02884225,  0.02720422, ..., -0.02438783,
        -0.0169282 , -0.00341779],
       ...,
       [ 0.01055255, -0.00945783, -0.00152049, ..., -0.03797672,
         0.01715031,  0.02279662],
       [ 0.03525571,  0.00862676,  0.02131652, ...,  0.00314574,
        -0.00221583,  0.01686604],
       [ 0.02903666, -0.03785585,  0.0386798 , ..., -0.00419276,
        -0.00648742, -0.02684806]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': Array([ 0.0026283 , -0.00116334,  0.0034741 , ..., -0.00103938,
        0.00077989,  0.00055067], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.6575837, 1.4231721, 1.9481301, ..., 1.6508639, 1.8229599,
        1.6238132]], dtype=float32),
                        'lora_a': Array([[-0.10849126],
       [ 0.29453638],
       [-0.26039052],
       ...,
       [-0.7982791 ],
       [ 0.73761576],
       [-0.08403185]], dtype=float32),
                        'lora_b': Array([[ 1.3248070e-05],
       [-5.6904032e-06],
       [ 1.3594767e-05],
       ...,
       [ 1.9065541e-05],
       [-1.5528435e-06],
       [-1.1918079e-05]], dtype=float32),
                        'w': Array([[ 4.3947481e-02,  4.8813175e-02,  7.3814997e-03, ...,
         5.8599636e-03,  3.8389336e-02, -4.0372342e-02],
       [ 3.5060907e-03, -2.8329035e-02,  2.6754966e-02, ...,
         6.0271759e-02,  4.4384837e-02, -4.1161864e-03],
       [ 1.5060810e-02,  4.7639436e-03, -6.8287635e-03, ...,
         8.3255162e-03, -2.5526772e-03,  9.9604297e-03],
       ...,
       [-2.9504925e-02,  2.0276491e-02,  4.6000529e-02, ...,
        -1.9012368e-04,  3.0562451e-02,  4.3479633e-02],
       [ 2.4119338e-02, -8.0203616e-05, -1.5165000e-02, ...,
         1.1062592e-02,  2.3118423e-03, -7.7793556e-03],
       [ 3.0921111e-02,  3.3886950e-02,  4.4420842e-02, ...,
        -1.1219447e-02,  1.9637524e-04,  1.2746536e-02]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': Array([ 0.00490241,  0.00695948,  0.00014633, ..., -0.00513277,
       -0.00615356,  0.00304212], dtype=float32),
                    'scale': Array([-0.00079224,  0.00206569, -0.01471212, ...,  0.00290086,
        0.0069622 , -0.00268389], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': Array([1.0908813, 1.1087157, 1.0528654, ..., 1.0645745, 1.0824969,
       1.0966488], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': Array([[-2.3981717e-03,  1.8856628e-03, -5.0984562e-04, ...,
         7.3413196e-04,  2.2099728e-03, -6.8922009e-04],
       [-2.7342146e-04, -2.5460121e-04, -2.0622846e-03, ...,
        -1.6955294e-03, -6.1627716e-04,  6.4225256e-04],
       [-4.1961134e-03,  1.7308042e-04,  2.5284584e-04, ...,
         1.2709869e-03,  3.9050930e-05,  7.5965037e-04],
       ...,
       [ 5.3820893e-04,  2.3627002e-03, -8.7761186e-04, ...,
         2.3640343e-03, -9.1595820e-04,  2.5802132e-04],
       [ 1.3188351e-03, -1.7600934e-03, -2.8453977e-04, ...,
         3.7302570e-03, -1.5706382e-03, -1.0020508e-03],
       [-1.8914823e-03, -7.5478945e-04, -2.6961186e-03, ...,
        -4.9656915e-04, -1.6461979e-03,  3.2195733e-03]], dtype=float32),
                    'dora_m': Array([[[0.40431568, 0.40140972, 0.47194266, ..., 0.49030623,
         0.4346639 , 0.46966082],
        [0.56254756, 0.5548262 , 0.54131144, ..., 0.44928303,
         0.5575844 , 0.5159123 ],
        [0.45118186, 0.4992576 , 0.41844377, ..., 0.4530989 ,
         0.41886905, 0.45597303],
        ...,
        [0.53227514, 0.5394606 , 0.49734318, ..., 0.505995  ,
         0.41433886, 0.46287405],
        [0.49171653, 0.43603128, 0.50175965, ..., 0.4215645 ,
         0.49965265, 0.47792813],
        [0.5605003 , 0.5434671 , 0.48735762, ..., 0.5007764 ,
         0.4222924 , 0.43579316]]], dtype=float32),
                    'lora_a': Array([[-0.10848374],
       [ 0.2945304 ],
       [-0.2603654 ],
       ...,
       [-0.79824406],
       [ 0.7376394 ],
       [-0.08400537]], dtype=float32),
                    'lora_b': Array([[ 9.2074270e-06],
       [-1.5770336e-05],
       [ 8.1538847e-06],
       ...,
       [ 8.9507512e-06],
       [-1.3893864e-05],
       [-5.5743094e-06]], dtype=float32),
                    'w': Array([[[ 6.40782015e-03, -3.66740656e-04,  5.17848786e-03, ...,
          1.36679262e-02, -3.99049465e-03,  6.54312270e-03],
        [ 1.33482981e-02, -5.79426842e-06,  1.14805858e-04, ...,
          2.80219043e-04, -2.07124953e-03, -1.47480005e-02],
        [ 9.00816638e-03, -3.81786795e-03,  5.01242885e-03, ...,
         -7.67782563e-03, -4.90695750e-03, -9.86438617e-03],
        ...,
        [-2.30965801e-02, -2.56713610e-02, -2.57862173e-02, ...,
         -3.67835164e-04,  3.84535571e-03, -1.73336025e-02],
        [-1.53349172e-02, -1.23835681e-02, -1.58335790e-02, ...,
          1.68166608e-02, -1.33521706e-02, -8.03984515e-03],
        [ 9.64884926e-03,  2.03878060e-02,  1.40785463e-02, ...,
          4.21672575e-02,  2.65351832e-02, -9.15440917e-03]],

...,

[[-1.41176418e-03,  1.28303468e-02,  1.05975266e-03, ...,
          1.96604617e-02,  2.04978511e-02,  2.20396370e-03],
        [-2.38677618e-04,  1.28345722e-02,  9.21138446e-04, ...,
          2.24298518e-02,  1.20315086e-02, -2.85336189e-02],
        [ 1.73810627e-02,  3.50343733e-04,  4.90271952e-03, ...,
         -1.35342497e-02, -2.17246432e-02, -2.02825442e-02],
        ...,
        [-9.97872464e-03, -3.68483714e-03,  1.12736169e-02, ...,
          8.30859691e-03,  4.30642674e-03, -5.83485328e-03],
        [-2.17700992e-02,  9.59218945e-03, -1.46980500e-02, ...,
          4.30577202e-03,  5.33139473e-03,  3.57369259e-02],
        [ 1.92339625e-03, -1.11738732e-02,  9.66002641e-04, ...,
         -6.37061149e-03, -3.23438871e-04,  6.01156382e-03]]],      dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': Array([0.1745563 , 0.17456126, 0.14752585, 0.15875179, 0.1832802 ,
       0.17058335, 0.13296197, 0.17121601, 0.10767658, 0.15312862,
       0.16568439, 0.1583921 , 0.1848713 , 0.1689837 , 0.16452795,
       0.16034178, 0.16015925, 0.18659249, 0.19397616, 0.16274922,
       0.1499158 , 0.16354154, 0.18811893, 0.16714635, 0.15575878,
       0.169097  , 0.17320454, 0.16842654, 0.18550313, 0.16002789,
       0.14593089, 0.18536483, 0.16758567, 0.12782294, 0.16751379,
       0.18410124, 0.13181742, 0.19628252, 0.18065505, 0.16620027,
       0.16837107, 0.17566574, 0.1590769 , 0.16995479, 0.15416765,
       0.17122333, 0.2122131 , 0.17859052, 0.18375066, 0.15738262,
       0.16987279, 0.1377341 , 0.14713049, 0.19109888, 0.16073835,
       0.155179  , 0.18291654, 0.15882567, 0.1332647 , 0.14301917,
       0.16009846, 0.13951255, 0.15479031, 0.17047878, 0.159181  ,
       0.12073389, 0.13865942, 0.16203314, 0.1777702 , 0.15428174,
       0.16125916, 0.1845427 , 0.1468957 , 0.17631026, 0.13524841,
       0.12261588, 0.12875122, 0.16690533, 0.16385828, 0.16424495],      dtype=float32)
                },
                'post': {
                    'b': Array([ 0.00290152,  0.00115534,  0.00654437, ..., -0.0028905 ,
        0.0003559 ,  0.00298951], dtype=float32),
                    'dora_m': Array([[[0.37762478, 0.40521967, 0.3820069 , ..., 0.39485344,
         0.372892  , 0.3839794 ],
        [0.38457096, 0.3861344 , 0.3855786 , ..., 0.3758471 ,
         0.37628263, 0.39046243],
        [0.38694894, 0.40024152, 0.37717003, ..., 0.4095392 ,
         0.39641118, 0.40640843],
        ...,
        [0.39896756, 0.428598  , 0.3719614 , ..., 0.38706172,
         0.39260438, 0.37393555],
        [0.37640527, 0.35111472, 0.38988966, ..., 0.39640877,
         0.37819082, 0.377096  ],
        [0.36173895, 0.34664425, 0.39248618, ..., 0.3894728 ,
         0.3908242 , 0.36472812]]], dtype=float32),
                    'lora_a': Array([[-0.10849071],
       [ 0.29455328],
       [-0.26037768],
       ...,
       [-0.7982553 ],
       [ 0.73762494],
       [-0.08401011]], dtype=float32),
                    'lora_b': Array([[ 1.4735085e-05],
       [ 2.1063524e-06],
       [ 1.5136572e-05],
       ...,
       [-2.0834743e-06],
       [ 1.7859180e-05],
       [ 6.9779408e-06]], dtype=float32),
                    'w': Array([[[ 1.02901012e-02,  1.41829038e-02, -1.01600485e-02, ...,
          6.96739834e-03,  1.74732395e-02, -3.62850353e-03],
        [ 1.74780264e-02,  1.75960902e-02, -7.69804139e-03, ...,
         -1.47320535e-02, -1.33710250e-03,  3.97499353e-02],
        [-7.04897457e-06,  2.79059093e-02,  1.67463962e-02, ...,
          1.56734269e-02,  6.61494210e-04, -8.48551840e-03],
        ...,
        [ 1.02333287e-02,  3.67542845e-03, -1.01666525e-02, ...,
          1.26873637e-02,  8.48016702e-03,  1.77150797e-02],
        [ 8.24190117e-03,  5.60097629e-03,  1.18684843e-02, ...,
          2.06912514e-02,  2.93256715e-03, -7.79098505e-03],
        [-1.25470618e-02, -1.41636077e-02, -2.62559578e-03, ...,
         -7.30138179e-03,  4.54861205e-03, -1.10310428e-02]],

...,

[[-8.00714921e-03,  1.30047426e-02,  1.02177646e-03, ...,
         -4.53452085e-04, -1.41068490e-03, -4.64247027e-03],
        [ 8.46171845e-03,  7.21889921e-03, -1.17666358e-02, ...,
         -8.61446839e-03, -1.08559737e-02,  8.59782379e-03],
        [-4.55562910e-03,  3.41780542e-04, -3.09505779e-03, ...,
          1.27979526e-02, -9.39201564e-03,  1.39818527e-02],
        ...,
        [ 3.17281141e-04, -8.64622369e-03,  3.42501700e-03, ...,
          2.67034722e-03, -4.23402665e-03,  2.99277948e-03],
        [-1.84059367e-02, -4.27844189e-03,  8.56903195e-03, ...,
          2.37914166e-04, -9.01622791e-03,  1.79223046e-02],
        [-9.79379378e-03,  1.14735682e-02,  1.30470167e-03, ...,
         -3.09733092e-04, -1.67014338e-02,  3.92003683e-03]]],      dtype=float32)
                },
                'query': {
                    'b': Array([[ 0.01523319, -0.00828504, -0.00129177, ...,  0.00156883,
        -0.00982505, -0.00408466],
       [-0.00088482, -0.00184363,  0.00965276, ...,  0.00431157,
        -0.00666857, -0.00053838],
       [-0.01299675,  0.00013154,  0.02363259, ..., -0.0216933 ,
         0.00750832,  0.01218794],
       ...,
       [-0.00368615,  0.00494643,  0.00254787, ..., -0.00205449,
         0.01362027,  0.02188244],
       [ 0.01115793, -0.00408845,  0.00487758, ...,  0.00150502,
        -0.00290738, -0.0086401 ],
       [ 0.00153861, -0.00713248, -0.00014092, ..., -0.00658943,
        -0.01316522, -0.00693501]], dtype=float32),
                    'dora_m': Array([[[0.5372839 , 0.50047016, 0.5274571 , ..., 0.5501638 ,
         0.61554074, 0.51553696],
        [0.6123375 , 0.60879344, 0.7656021 , ..., 0.7650865 ,
         0.66413987, 0.64069617],
        [0.6596833 , 0.6859899 , 0.61169   , ..., 0.6156229 ,
         0.6862928 , 0.7413911 ],
        ...,
        [0.8399897 , 0.6725385 , 0.7911484 , ..., 0.7149487 ,
         0.72369766, 0.8656877 ],
        [0.6051752 , 0.50430065, 0.6335697 , ..., 0.5442369 ,
         0.5918453 , 0.61850226],
        [0.58660126, 0.65712416, 0.67801154, ..., 0.6816402 ,
         0.58229744, 0.5377895 ]]], dtype=float32),
                    'lora_a': Array([[-0.10848489],
       [ 0.29451588],
       [-0.2603749 ],
       ...,
       [-0.79827505],
       [ 0.7376182 ],
       [-0.08403634]], dtype=float32),
                    'lora_b': Array([[-2.6108996e-06],
       [-2.0184043e-05],
       [ 1.8134775e-05],
       ...,
       [-2.0412870e-05],
       [-1.6797159e-05],
       [-2.0477260e-05]], dtype=float32),
                    'w': Array([[[-1.13905529e-02, -3.67759634e-03, -2.04983577e-02, ...,
          1.11687044e-02,  5.42940281e-04,  1.82931740e-02],
        [ 1.40451295e-02, -9.68987588e-03,  2.07039956e-02, ...,
          3.94661725e-02,  1.94060840e-02,  1.49192326e-02],
        [-7.23185530e-03,  1.94731783e-02,  3.10550374e-03, ...,
         -2.07566041e-02,  3.11420206e-03,  7.74376141e-03],
        ...,
        [-1.05216971e-03, -2.06391374e-03, -3.61125767e-02, ...,
         -2.35695615e-02, -1.78337395e-02,  1.17942831e-02],
        [ 9.29886755e-03,  2.45120320e-02,  7.42621627e-03, ...,
          1.73580907e-02, -7.13619683e-03, -2.02737600e-02],
        [ 6.38788752e-03,  1.45135447e-02,  5.60244266e-03, ...,
         -1.47173805e-02, -6.62107253e-03,  1.58582255e-02]],

...,

[[-1.50356777e-02,  5.00446605e-03, -1.51625159e-03, ...,
          4.74293064e-03,  2.27355901e-02, -6.27850043e-03],
        [ 4.54321653e-02, -2.74610631e-02, -4.70345244e-02, ...,
          8.63380730e-03,  4.13595140e-02,  2.94508738e-03],
        [-1.30670536e-02, -2.23860908e-02,  4.23285691e-03, ...,
         -3.14619625e-04, -4.14622389e-02,  3.56040038e-02],
        ...,
        [-5.40730469e-02, -1.84541214e-02,  2.73042731e-02, ...,
         -2.20951382e-02,  2.24010423e-02, -1.97337382e-02],
        [ 7.33314129e-03,  1.78060699e-02, -2.36569867e-02, ...,
         -8.27676803e-03,  2.98645347e-02, -1.18271064e-03],
        [-1.29001159e-02,  1.34417852e-02,  2.15053628e-03, ...,
         -2.29821657e-03,  2.24603470e-02, -6.90755760e-03]]],      dtype=float32)
                },
                'value': {
                    'b': Array([[ 1.5917098e-03,  1.4896401e-03, -2.0226375e-03, ...,
        -1.7325892e-03,  2.9298672e-03, -2.7322539e-03],
       [-3.1144351e-03, -4.1114488e-03,  1.0012481e-03, ...,
         4.8245918e-03,  6.4365431e-03, -1.2060608e-03],
       [ 4.5222994e-03,  9.4918645e-04, -6.6103740e-04, ...,
         1.8348355e-03,  3.3785610e-03, -7.6923724e-03],
       ...,
       [-7.8128959e-04,  1.6019873e-03,  8.1217266e-04, ...,
        -4.9950148e-04, -2.2263450e-03, -1.9558065e-03],
       [-1.9730320e-03, -1.4390795e-03,  1.7571831e-03, ...,
         3.8916760e-03, -2.0562236e-03, -1.1587031e-03],
       [-3.9481013e-03,  3.2905554e-03,  4.4394241e-05, ...,
         2.0804864e-03, -5.7885433e-03,  3.5198012e-03]], dtype=float32),
                    'dora_m': Array([[[0.448648  , 0.47045946, 0.43834075, ..., 0.44747895,
         0.43676138, 0.4557012 ],
        [0.4217106 , 0.4477234 , 0.4449377 , ..., 0.4187558 ,
         0.4297295 , 0.46097842],
        [0.44931585, 0.47595847, 0.45715868, ..., 0.46125057,
         0.46057963, 0.46046388],
        ...,
        [0.46648684, 0.48288277, 0.43108726, ..., 0.44995221,
         0.44506064, 0.40819466],
        [0.43695417, 0.4243253 , 0.46251372, ..., 0.45483315,
         0.4334071 , 0.45308554],
        [0.41338536, 0.42687193, 0.48991957, ..., 0.45207494,
         0.45824972, 0.43513566]]], dtype=float32),
                    'lora_a': Array([[-0.10849072],
       [ 0.29451442],
       [-0.26038176],
       ...,
       [-0.798278  ],
       [ 0.73761415],
       [-0.08404024]], dtype=float32),
                    'lora_b': Array([[ 1.9705056e-05],
       [ 1.1415477e-05],
       [-9.9789095e-06],
       ...,
       [ 7.8308894e-06],
       [-9.1436741e-06],
       [ 1.2796164e-05]], dtype=float32),
                    'w': Array([[[-1.60310082e-02, -9.71163157e-03, -1.20064905e-02, ...,
         -1.51673928e-02, -8.96296173e-04, -6.39644917e-03],
        [-1.70423854e-02, -6.60409639e-03,  7.88445026e-03, ...,
          2.02861018e-02, -2.18733568e-02, -2.00053845e-02],
        [ 4.50440589e-03, -8.13141465e-03,  1.07267508e-04, ...,
          7.52942683e-03,  1.85728399e-03,  9.87506006e-03],
        ...,
        [-1.12143280e-02, -3.10050487e-03,  1.23486603e-02, ...,
         -4.45978865e-02, -7.39333080e-03, -6.61169877e-03],
        [-1.48295267e-02, -5.04418463e-03, -1.81792956e-02, ...,
         -1.82359703e-02, -6.24101004e-03,  6.44446258e-03],
        [ 1.23566040e-03, -5.22012264e-03,  3.13698314e-03, ...,
          1.32481931e-02,  6.28043199e-03, -4.75604553e-03]],

...,

[[-1.17207430e-02, -1.41995978e-02, -1.24308169e-02, ...,
         -1.09773008e-02, -6.52138842e-03, -4.35515959e-03],
        [ 2.22678925e-03,  7.40520703e-03, -1.84388403e-02, ...,
          4.69964603e-03,  2.42842291e-03, -2.51398217e-02],
        [ 3.94491130e-04,  1.39072891e-02,  1.91890746e-02, ...,
         -5.70005318e-03,  2.28017895e-03, -9.48434789e-03],
        ...,
        [-2.89604533e-03,  1.19845793e-02, -9.42718890e-03, ...,
          1.52631383e-02, -5.83796343e-03, -1.43370975e-03],
        [ 9.60272551e-03, -1.21000046e-02,  8.05047061e-03, ...,
         -1.12463962e-02,  9.17629153e-03, -8.07019940e-04],
        [-3.64702418e-02, -1.07744811e-02,  1.80233661e-02, ...,
         -1.90189574e-02,  1.28748957e-02, -7.23076938e-03]]],      dtype=float32)
                }
            }
        },
        'x_layers_8': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': Array([-0.00671606, -0.00744732, -0.01157051, ..., -0.00302582,
       -0.00751212, -0.00731062], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.0048492, 1.0787207, 1.0907636, ..., 1.1749911, 1.1022582,
        1.0469828]], dtype=float32),
                        'lora_a': Array([[-0.10848572],
       [ 0.2945511 ],
       [-0.26038817],
       ...,
       [-0.79826653],
       [ 0.7376151 ],
       [-0.08403418]], dtype=float32),
                        'lora_b': Array([[-2.1404963e-05],
       [ 1.3854777e-05],
       [ 3.6645129e-06],
       ...,
       [-1.5139792e-05],
       [ 3.6818285e-06],
       [ 1.9569168e-05]], dtype=float32),
                        'w': Array([[-0.01811136, -0.04249045, -0.02398836, ...,  0.03573981,
        -0.06078571, -0.00195698],
       [ 0.04663501, -0.01439486,  0.00278891, ...,  0.01190364,
         0.03039053, -0.02769112],
       [-0.02636963,  0.03524577,  0.02645216, ..., -0.04963003,
         0.02481861, -0.01761094],
       ...,
       [ 0.03440456, -0.05879504, -0.0126651 , ...,  0.05787901,
        -0.03594774, -0.0137208 ],
       [ 0.00762873, -0.01527641, -0.00911149, ..., -0.00852254,
         0.01500743, -0.00605547],
       [ 0.0276429 ,  0.00140306,  0.02717231, ...,  0.00757653,
        -0.05295376, -0.0348709 ]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': Array([ 1.7641198e-03,  7.2908530e-05,  2.7227844e-03, ...,
       -7.4495800e-04, -3.3278845e-04,  2.1731786e-03], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.5409466, 1.407298 , 1.8424067, ..., 1.5180651, 1.7194695,
        1.700189 ]], dtype=float32),
                        'lora_a': Array([[-0.10846411],
       [ 0.29455253],
       [-0.26038554],
       ...,
       [-0.79826844],
       [ 0.73763925],
       [-0.08400098]], dtype=float32),
                        'lora_b': Array([[ 5.7391858e-06],
       [ 5.9182876e-06],
       [-1.2349833e-05],
       ...,
       [ 5.2171604e-06],
       [ 8.7914250e-06],
       [ 1.5592599e-05]], dtype=float32),
                        'w': Array([[-0.03619846,  0.01551942,  0.0302618 , ...,  0.00047247,
        -0.01799299,  0.04210168],
       [ 0.03249831, -0.0137839 ,  0.04283076, ...,  0.01192124,
         0.01020726,  0.01523312],
       [-0.0461726 , -0.00367335,  0.03881538, ...,  0.03333255,
         0.05790588,  0.00927986],
       ...,
       [ 0.00713237,  0.00093277,  0.00547557, ..., -0.02959165,
        -0.01509296, -0.03668424],
       [ 0.0416071 , -0.01149079,  0.06314063, ...,  0.05279715,
        -0.02821371, -0.02391061],
       [-0.03470379, -0.03203568,  0.01379244, ..., -0.01838429,
         0.03363138,  0.02721118]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': Array([ 0.00412433,  0.00395123,  0.00951787, ..., -0.00058707,
       -0.00273577,  0.00690942], dtype=float32),
                    'scale': Array([-0.0213481 ,  0.00577652,  0.00104609, ..., -0.00437833,
        0.00163347,  0.01052476], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': Array([1.1052154, 1.1062342, 1.0680627, ..., 1.0658808, 1.0893704,
       1.1054778], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': Array([[ 2.0952977e-03,  1.5934280e-03,  1.4717099e-03, ...,
        -1.6025685e-03, -1.7999718e-03, -3.2979084e-04],
       [-2.6650492e-03,  1.3669929e-03,  3.8853692e-04, ...,
         4.1401391e-03, -8.2425791e-04, -1.0697824e-03],
       [-1.2259793e-03, -2.6739226e-03, -5.6987611e-04, ...,
        -7.4962771e-04,  1.7805287e-03,  8.2909531e-04],
       ...,
       [-7.6656679e-06,  9.1410882e-04, -2.2895208e-03, ...,
        -1.3394810e-03, -4.1470411e-03, -9.1867056e-04],
       [-1.6828876e-03,  7.3032081e-04, -3.9617313e-04, ...,
        -7.6685828e-06, -1.3311633e-03, -8.0681883e-04],
       [ 5.7779968e-04,  1.3798820e-04,  1.4514620e-03, ...,
        -9.1995212e-04,  1.8731453e-05, -9.3555247e-04]], dtype=float32),
                    'dora_m': Array([[[0.4180446 , 0.5161904 , 0.4386738 , ..., 0.50602263,
         0.44910067, 0.51460975],
        [0.4286451 , 0.44221836, 0.4611636 , ..., 0.56178576,
         0.54498327, 0.44523278],
        [0.4471568 , 0.50054914, 0.5529091 , ..., 0.604724  ,
         0.52428883, 0.50026643],
        ...,
        [0.4542235 , 0.5157331 , 0.48553276, ..., 0.44519523,
         0.41363624, 0.44002497],
        [0.5069172 , 0.48069718, 0.5083314 , ..., 0.45221925,
         0.48456642, 0.5178936 ],
        [0.39659387, 0.46775395, 0.47984323, ..., 0.49478525,
         0.44235218, 0.44521523]]], dtype=float32),
                    'lora_a': Array([[-0.1084741 ],
       [ 0.29452336],
       [-0.26036617],
       ...,
       [-0.79824567],
       [ 0.7376222 ],
       [-0.08403409]], dtype=float32),
                    'lora_b': Array([[ 3.9901934e-06],
       [-1.2444397e-06],
       [ 1.0598664e-05],
       ...,
       [-2.1101823e-06],
       [ 9.4554416e-06],
       [-4.7303242e-06]], dtype=float32),
                    'w': Array([[[-0.00633128,  0.00398494,  0.01548496, ..., -0.00509497,
          0.00089846,  0.00314374],
        [-0.00360716, -0.00279797, -0.03103199, ..., -0.0191979 ,
          0.00687667, -0.00903005],
        [ 0.00590024,  0.00502111, -0.00143865, ...,  0.00133199,
          0.0051042 , -0.01144267],
        ...,
        [-0.0215076 , -0.01586042, -0.02274709, ...,  0.01589907,
          0.01593778, -0.0125594 ],
        [ 0.0002792 , -0.00019325, -0.02181054, ..., -0.00606402,
          0.00556961, -0.00287801],
        [ 0.01454523, -0.0023863 , -0.00274182, ...,  0.00334038,
         -0.00116689,  0.00762898]],

...,

[[-0.0001781 , -0.01084789,  0.00013601, ..., -0.00540317,
         -0.00850195, -0.02196166],
        [ 0.01238866,  0.00744494, -0.01020275, ...,  0.00031833,
          0.01465734,  0.01575067],
        [-0.00223196, -0.00378659, -0.00145925, ..., -0.00451101,
          0.00256217, -0.00203178],
        ...,
        [ 0.02033761,  0.00679272, -0.00348961, ...,  0.00539042,
          0.01451758,  0.00278541],
        [ 0.00109349,  0.00856665, -0.00203083, ...,  0.01030129,
          0.01096501,  0.00195871],
        [ 0.01040758, -0.01207248,  0.02535666, ...,  0.01254702,
          0.01569652, -0.01007556]]], dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': Array([0.16659452, 0.2176413 , 0.20913084, 0.16684686, 0.14747609,
       0.13816825, 0.17060304, 0.1732533 , 0.18752515, 0.18463098,
       0.17490867, 0.17087072, 0.20064996, 0.12334649, 0.20583531,
       0.20181543, 0.14958781, 0.14743958, 0.14016852, 0.13809851,
       0.14118049, 0.13237341, 0.15868132, 0.13771881, 0.16029447,
       0.14321326, 0.15923475, 0.17038305, 0.14117105, 0.16902876,
       0.14414205, 0.0990934 , 0.14415134, 0.17895143, 0.16005166,
       0.1644718 , 0.15972514, 0.16798589, 0.16425624, 0.17587021,
       0.15190102, 0.18830405, 0.18460017, 0.13043234, 0.2084825 ,
       0.15443006, 0.1792815 , 0.21106526, 0.16622984, 0.15344281,
       0.17172691, 0.18035972, 0.15648267, 0.19878417, 0.14236848,
       0.19366716, 0.15613955, 0.12001559, 0.1338472 , 0.20615734,
       0.14170356, 0.18917897, 0.14692408, 0.13729933, 0.2014039 ,
       0.1762525 , 0.17112777, 0.19586515, 0.14007089, 0.15948854,
       0.11711452, 0.14105257, 0.1645794 , 0.12344601, 0.1573255 ,
       0.17944992, 0.12884043, 0.15109557, 0.12746932, 0.19371282],      dtype=float32)
                },
                'post': {
                    'b': Array([ 0.00145101,  0.00251218,  0.00792553, ..., -0.00019417,
       -0.00048331,  0.00213577], dtype=float32),
                    'dora_m': Array([[[0.40835515, 0.3991751 , 0.40936375, ..., 0.41190213,
         0.3968558 , 0.4171563 ],
        [0.3940997 , 0.38256684, 0.38246635, ..., 0.39678723,
         0.38526696, 0.36888734],
        [0.40547502, 0.3997352 , 0.3984087 , ..., 0.4163389 ,
         0.3949618 , 0.40609616],
        ...,
        [0.36374435, 0.37965506, 0.3697335 , ..., 0.37624633,
         0.35164478, 0.39694703],
        [0.37595648, 0.41303852, 0.42157325, ..., 0.38356978,
         0.40332028, 0.4093307 ],
        [0.39595228, 0.39538616, 0.38260627, ..., 0.40435058,
         0.39198273, 0.40686658]]], dtype=float32),
                    'lora_a': Array([[-0.1084908 ],
       [ 0.29455307],
       [-0.26038724],
       ...,
       [-0.79827   ],
       [ 0.73765063],
       [-0.08401109]], dtype=float32),
                    'lora_b': Array([[ 1.9928113e-06],
       [ 1.0106560e-05],
       [ 1.3300108e-05],
       ...,
       [-1.0778340e-05],
       [-1.5361049e-05],
       [-1.2469696e-05]], dtype=float32),
                    'w': Array([[[ 1.27353165e-02, -3.06770834e-03, -5.33052022e-03, ...,
         -4.73721186e-03,  1.25906803e-02, -1.32778967e-02],
        [-1.10966377e-02, -1.76666062e-02,  4.20981180e-03, ...,
          9.97211505e-03,  6.93971943e-03,  6.30564895e-03],
        [ 4.62995982e-03, -2.66032148e-04,  2.08375137e-03, ...,
         -9.29566100e-03,  8.34042300e-03, -6.12970488e-03],
        ...,
        [ 5.17173670e-03,  4.32177912e-03, -1.01235826e-02, ...,
          1.53043903e-02,  1.35354849e-03,  9.87807103e-03],
        [ 6.90858811e-03,  1.98646523e-02, -2.52215681e-03, ...,
          7.05157081e-03, -1.67272799e-02, -3.48131009e-03],
        [ 1.39710959e-02, -1.07559208e-02, -2.13850965e-03, ...,
          9.27119167e-04, -3.73881147e-03,  8.89662828e-04]],

...,

[[ 9.04358551e-03,  1.56450942e-02, -3.21527361e-03, ...,
          3.07923579e-03,  2.93071754e-03, -8.14707112e-03],
        [-5.97499078e-03, -1.44700357e-03, -3.24818585e-03, ...,
          1.24646854e-02,  7.60318292e-03, -1.23952399e-03],
        [-1.11762225e-03, -3.22844251e-03, -9.97110456e-03, ...,
          1.02657862e-02, -8.27095658e-03,  1.30888466e-02],
        ...,
        [ 6.26176689e-03,  2.16218941e-02, -5.57345664e-03, ...,
         -2.45052413e-03,  2.03148648e-03,  5.85137022e-05],
        [-8.74185003e-04,  2.17746552e-02, -1.16565423e-02, ...,
         -1.39208622e-02, -3.95378703e-03,  1.49394246e-02],
        [-1.21912425e-02, -2.50114570e-03, -8.13443586e-03, ...,
         -1.52346827e-02, -2.04536365e-03,  4.48474661e-03]]],      dtype=float32)
                },
                'query': {
                    'b': Array([[ 2.4137442e-04,  5.9842686e-03,  2.4649238e-02, ...,
        -1.8480065e-03, -1.1731535e-02,  1.5507266e-03],
       [-2.3323870e-03, -2.1887743e-03,  1.6048562e-03, ...,
        -1.4903965e-02, -1.6681623e-02,  1.1353353e-03],
       [ 2.3728411e-03,  7.1505019e-03,  5.7950812e-05, ...,
         4.6056965e-03, -8.6643379e-03,  6.6809235e-03],
       ...,
       [-1.4967571e-02, -9.4790878e-03, -1.0470150e-02, ...,
         1.3970692e-02,  3.4786370e-03, -3.5595410e-03],
       [ 8.3640805e-03,  1.7302636e-02,  7.8781396e-03, ...,
         1.6122483e-02,  9.5468238e-03, -2.2607304e-03],
       [-2.2019206e-02,  3.1206079e-03,  6.5784846e-03, ...,
        -1.1271449e-02, -1.5769789e-02,  1.5105515e-02]], dtype=float32),
                    'dora_m': Array([[[0.8154983 , 0.82559425, 0.8244399 , ..., 0.79515934,
         0.82818145, 0.69265294],
        [0.5422536 , 0.6026471 , 0.612632  , ..., 0.52474976,
         0.5280597 , 0.5955887 ],
        [0.6128743 , 0.84549713, 0.5525184 , ..., 0.58983505,
         0.58330846, 0.8413922 ],
        ...,
        [0.6336001 , 0.5334065 , 0.7261618 , ..., 0.58794665,
         0.5971227 , 0.57848823],
        [0.7854873 , 0.87135607, 0.87839234, ..., 0.78517306,
         0.60140944, 0.791944  ],
        [0.566211  , 0.6046119 , 0.5934606 , ..., 0.71311414,
         0.68624514, 0.66695565]]], dtype=float32),
                    'lora_a': Array([[-0.10848746],
       [ 0.29453668],
       [-0.26038647],
       ...,
       [-0.79827523],
       [ 0.73761654],
       [-0.08402876]], dtype=float32),
                    'lora_b': Array([[-9.1158063e-06],
       [-1.7000890e-05],
       [ 1.1507009e-05],
       ...,
       [ 1.6050304e-05],
       [ 9.5951664e-06],
       [-1.8843006e-05]], dtype=float32),
                    'w': Array([[[-1.83182843e-02,  1.23743725e-03,  2.92542875e-02, ...,
         -3.55846807e-02, -9.71184485e-03,  1.23744260e-03],
        [ 3.53889056e-02, -2.28063986e-02, -2.86072167e-03, ...,
         -1.58963613e-02, -1.60532327e-05, -8.02235119e-03],
        [-1.85477771e-02, -4.17265072e-02, -1.83387920e-02, ...,
         -4.50981036e-03, -1.07226791e-02, -2.05805842e-02],
        ...,
        [-1.31660281e-02, -1.52303940e-02, -1.63004815e-03, ...,
          5.14348224e-03,  9.24873538e-03, -2.81867501e-03],
        [-3.78815550e-03, -5.74611276e-02, -1.33208502e-02, ...,
          3.44731589e-03, -2.08448209e-02, -1.28401117e-02],
        [ 7.96766113e-03,  1.09075643e-02, -8.05056933e-03, ...,
         -1.16875744e-03, -1.54173728e-02,  3.37275192e-02]],

...,

[[ 1.67931709e-02, -3.89110134e-03, -7.07024755e-03, ...,
         -1.75038241e-02, -1.41046401e-02, -1.22451773e-02],
        [-2.25986191e-03,  2.28019506e-02, -1.65104698e-02, ...,
          1.21603906e-02,  7.39587331e-03, -1.51593311e-04],
        [ 2.36524432e-03,  8.21779016e-03,  1.16037857e-02, ...,
          2.04978175e-02, -8.24823976e-03, -8.35203566e-03],
        ...,
        [-1.06447320e-02, -5.12641557e-02, -4.48293984e-02, ...,
          3.25738192e-02,  3.45098414e-02, -1.49965817e-02],
        [ 2.10662968e-02, -3.54700051e-02, -2.07886472e-02, ...,
         -1.63861848e-02,  4.22230270e-03,  7.40205422e-02],
        [ 2.63525583e-02,  2.26179305e-02,  2.27909572e-02, ...,
          4.20264248e-03, -9.29573365e-03,  6.99693430e-03]]],      dtype=float32)
                },
                'value': {
                    'b': Array([[ 4.1770306e-03,  5.1761670e-03, -6.7285066e-03, ...,
         2.4542394e-03, -2.6138593e-03, -7.2140631e-04],
       [-7.5517694e-04, -3.9448631e-03, -2.1563931e-03, ...,
        -3.8056246e-03,  5.6819087e-03, -4.0685167e-03],
       [-3.4288532e-04, -4.0260023e-03, -5.0083274e-04, ...,
        -7.6099916e-04, -1.9242569e-03, -5.2494492e-04],
       ...,
       [-2.9573161e-03,  1.5531457e-03, -1.6761255e-03, ...,
        -2.6534407e-03,  3.9581796e-03,  9.5842370e-05],
       [ 3.3524251e-03,  6.2615072e-05,  2.0030276e-03, ...,
         2.8712992e-03, -2.8626940e-03, -2.0652276e-03],
       [ 3.6921471e-04,  3.5650653e-03,  9.7389205e-04, ...,
         9.9158031e-04, -1.2612826e-03, -8.0347230e-04]], dtype=float32),
                    'dora_m': Array([[[0.4454674 , 0.43204132, 0.43815193, ..., 0.4436402 ,
         0.43570864, 0.46934894],
        [0.4621237 , 0.41381645, 0.4428268 , ..., 0.45351905,
         0.43179804, 0.41830117],
        [0.47213063, 0.44415495, 0.46591774, ..., 0.48888034,
         0.47863516, 0.485627  ],
        ...,
        [0.41534534, 0.43948728, 0.4313458 , ..., 0.41959617,
         0.3984804 , 0.470627  ],
        [0.4331019 , 0.46372798, 0.485454  , ..., 0.43244204,
         0.48469508, 0.47178718],
        [0.47399643, 0.46686375, 0.45951775, ..., 0.46810678,
         0.47249675, 0.48010743]]], dtype=float32),
                    'lora_a': Array([[-0.10849108],
       [ 0.2945195 ],
       [-0.26039186],
       ...,
       [-0.7982777 ],
       [ 0.737614  ],
       [-0.08404025]], dtype=float32),
                    'lora_b': Array([[-7.8038929e-06],
       [-1.6611595e-05],
       [-1.9076999e-05],
       ...,
       [ 1.2928789e-05],
       [-5.6346266e-06],
       [ 1.8251218e-05]], dtype=float32),
                    'w': Array([[[-1.03718126e-02,  3.82453911e-02, -6.29646285e-03, ...,
          1.13670034e-02, -2.14453563e-02,  3.31411976e-03],
        [ 6.25608023e-03,  1.31760472e-02, -1.72357615e-02, ...,
         -2.18084734e-02,  4.69026039e-04,  1.76579040e-02],
        [-4.03056853e-03, -2.83134095e-02,  6.33138046e-03, ...,
          2.66450122e-02,  1.12062646e-02, -1.31124966e-02],
        ...,
        [-1.07738366e-02, -3.03360820e-03, -8.02534260e-03, ...,
         -1.35927927e-02, -7.88540300e-03,  2.46340707e-02],
        [-2.51693465e-02, -1.15373321e-02, -1.81824174e-02, ...,
          9.18297283e-03,  2.27255132e-02, -9.70875286e-03],
        [ 3.92003581e-02,  1.23058343e-02, -1.22627122e-02, ...,
         -3.73543636e-03, -1.97548363e-02, -3.19940806e-03]],

...,

[[ 4.40953998e-03,  5.04396996e-03,  1.78099032e-02, ...,
          5.27709723e-03, -4.73994808e-03, -9.23941657e-03],
        [ 8.66723433e-03,  1.35799265e-02, -4.54121828e-03, ...,
         -3.35079362e-03, -1.27472188e-02,  1.04994085e-02],
        [ 4.40144475e-04, -3.29202670e-03, -1.68872643e-02, ...,
         -3.02552562e-02, -3.17824702e-03,  6.67277956e-03],
        ...,
        [-1.29628535e-02, -2.33154222e-02,  1.43350307e-02, ...,
         -1.01879351e-02, -1.74827613e-02, -4.70098993e-03],
        [ 6.43897522e-03,  3.64626106e-03,  2.52360757e-03, ...,
          1.22200791e-02,  4.17962018e-03, -5.20875631e-03],
        [ 6.19279826e-03,  1.06170094e-02, -9.20233130e-03, ...,
         -8.31399206e-03, -1.35130780e-02, -9.58402082e-03]]],      dtype=float32)
                }
            }
        },
        'x_layers_9': {
            'ff_layer': {
                'ffn_layer1': {
                    'bias': {
                        'b': Array([-0.00398949,  0.00553328, -0.00854538, ..., -0.01065977,
       -0.01085383,  0.10406972], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.0556884, 0.9502567, 1.0434325, ..., 1.2208557, 1.0806694,
        1.8669682]], dtype=float32),
                        'lora_a': Array([[-0.10847617],
       [ 0.29455388],
       [-0.2603716 ],
       ...,
       [-0.79827845],
       [ 0.737614  ],
       [-0.08402438]], dtype=float32),
                        'lora_b': Array([[-1.3450109e-05],
       [-1.5491110e-05],
       [-3.1316182e-07],
       ...,
       [-8.8784645e-06],
       [ 8.7398494e-06],
       [ 0.0000000e+00]], dtype=float32),
                        'w': Array([[ 0.00385022,  0.06122993,  0.02721928, ..., -0.03239259,
         0.01958293, -0.09112478],
       [-0.02464781,  0.01411509,  0.04660855, ...,  0.02175984,
        -0.00267561,  0.02156737],
       [-0.01745151, -0.017796  ,  0.00316033, ..., -0.01512817,
        -0.03007369, -0.04826004],
       ...,
       [ 0.03673194,  0.00620448,  0.00960318, ...,  0.00170189,
         0.03893861,  0.05012967],
       [-0.03062727,  0.01344013,  0.02024613, ..., -0.00557454,
         0.03241096,  0.02492862],
       [ 0.00941322, -0.03152687,  0.01848996, ..., -0.00659775,
         0.04427981, -0.02963124]], dtype=float32)
                    }
                },
                'ffn_layer2': {
                    'bias': {
                        'b': Array([ 0.00206702,  0.00195731,  0.00262794, ...,  0.00019651,
       -0.00277163,  0.00101381], dtype=float32)
                    },
                    'linear': {
                        'dora_m': Array([[1.2591255, 1.3849531, 1.3253784, ..., 1.2145861, 1.3025317,
        1.478849 ]], dtype=float32),
                        'lora_a': Array([[-0.10847054],
       [ 0.2945533 ],
       [-0.26036835],
       ...,
       [-0.7982654 ],
       [ 0.7376341 ],
       [-0.08401953]], dtype=float32),
                        'lora_b': Array([[-1.6702454e-05],
       [ 1.9101621e-05],
       [-1.6181901e-05],
       ...,
       [ 1.5275897e-05],
       [-1.7587740e-06],
       [ 1.0456820e-05]], dtype=float32),
                        'w': Array([[ 1.3016735e-02, -1.3118952e-02,  1.3808155e-02, ...,
        -1.3593927e-02,  1.9679509e-02,  2.5111713e-02],
       [ 1.8518310e-02, -1.8335573e-02,  5.6840559e-03, ...,
        -1.7456638e-02,  3.6089368e-02,  4.5100108e-02],
       [ 8.2706856e-03,  6.8996758e-03, -8.6885365e-03, ...,
        -5.9069094e-04,  2.7520629e-03, -4.7562275e-02],
       ...,
       [-7.4123950e-03, -1.9565022e-02, -1.4083363e-03, ...,
         5.1535361e-02, -2.9559454e-02, -2.9456580e-02],
       [ 2.9814113e-02,  9.0444470e-03,  6.7100316e-02, ...,
        -4.8956252e-03,  2.9108735e-02, -9.5660544e-05],
       [-1.1483442e-01,  7.0538826e-02, -1.7947572e-01, ...,
        -6.6035509e-02,  1.1661811e-01, -6.2948383e-02]], dtype=float32)
                    }
                },
                'layer_norm': {
                    'bias': Array([ 0.00483031, -0.00146662,  0.00633642, ..., -0.00554054,
       -0.00470097, -0.0005318 ], dtype=float32),
                    'scale': Array([-0.00955576,  0.00869239,  0.00359508, ..., -0.0093105 ,
        0.00919607,  0.01382339], dtype=float32)
                }
            },
            'layer_norm': {
                'scale': Array([1.1348704, 1.1547747, 1.1233397, ..., 1.0905623, 1.0844481,
       1.1467605], dtype=float32)
            },
            'self_attention': {
                'key': {
                    'b': Array([[-0.0038408 ,  0.00185112, -0.00173835, ...,  0.00336161,
         0.00030514,  0.00082349],
       [ 0.00067013, -0.00466094,  0.0019564 , ..., -0.00179259,
        -0.00069546,  0.00018717],
       [ 0.00160742,  0.00054143, -0.00041573, ...,  0.00107178,
         0.00017538,  0.00085705],
       ...,
       [ 0.00180693,  0.0006135 ,  0.00234484, ...,  0.00308815,
        -0.00226742, -0.00088748],
       [-0.00088424,  0.00179963, -0.00305551, ..., -0.00693885,
         0.00103404,  0.0019422 ],
       [-0.0019314 , -0.00187426,  0.00245309, ..., -0.00048181,
        -0.00064726, -0.00287785]], dtype=float32),
                    'dora_m': Array([[[0.5445889 , 0.55336577, 0.5124322 , ..., 0.49040678,
         0.5211972 , 0.49621388],
        [0.5429595 , 0.47574455, 0.5288338 , ..., 0.49633938,
         0.5912846 , 0.5780891 ],
        [0.4494602 , 0.76933926, 0.55888337, ..., 0.5288396 ,
         0.5989316 , 0.45583415],
        ...,
        [0.39042372, 0.3447509 , 0.47431168, ..., 0.42211318,
         0.4263954 , 0.58565646],
        [0.5624337 , 0.44174156, 0.44325176, ..., 0.4278546 ,
         0.41590437, 0.5675641 ],
        [0.4596165 , 0.6124611 , 0.44578063, ..., 0.5026597 ,
         0.45535272, 0.48011547]]], dtype=float32),
                    'lora_a': Array([[-0.10847511],
       [ 0.29453045],
       [-0.260374  ],
       ...,
       [-0.79826707],
       [ 0.73764765],
       [-0.08401802]], dtype=float32),
                    'lora_b': Array([[ 1.59867523e-05],
       [-8.01959686e-06],
       [-1.26743635e-05],
       ...,
       [ 1.58319726e-05],
       [-1.37042643e-05],
       [ 1.81942687e-05]], dtype=float32),
                    'w': Array([[[ 0.012124  ,  0.00197903,  0.01721361, ..., -0.01422858,
         -0.00852192,  0.00429994],
        [-0.00209467,  0.0140839 , -0.02325453, ..., -0.00315203,
         -0.01315125,  0.02884499],
        [ 0.00509635, -0.03448816,  0.01469674, ...,  0.00184527,
          0.00365957, -0.01060968],
        ...,
        [ 0.00186667, -0.00185121, -0.00026951, ..., -0.00374656,
          0.01521753,  0.00336414],
        [-0.02210866, -0.00564335, -0.00732728, ..., -0.01133978,
          0.00138814, -0.00057869],
        [-0.0173609 , -0.03433024,  0.02274247, ..., -0.00359569,
         -0.01011684, -0.01354941]],

...,

[[ 0.0008972 ,  0.00157054,  0.0047218 , ..., -0.01568508,
          0.01023996,  0.01188735],
        [-0.01286612, -0.02371632, -0.00060593, ...,  0.01111791,
          0.00755575, -0.02098973],
        [-0.02322051, -0.04816492, -0.00661085, ..., -0.00524904,
         -0.00385692, -0.01056321],
        ...,
        [ 0.00430305, -0.00331239,  0.00269268, ..., -0.00651208,
         -0.01047816,  0.00297151],
        [ 0.01365148,  0.01276014, -0.00482784, ..., -0.0023671 ,
         -0.01253744, -0.01830389],
        [ 0.00691162, -0.0166491 ,  0.00581269, ...,  0.02547496,
         -0.01401114, -0.00315702]]], dtype=float32)
                },
                'per_dim_scale': {
                    'per_dim_scale': Array([0.238903  , 0.22793801, 0.23768473, 0.2100576 , 0.23713684,
       0.27073467, 0.26104686, 0.2744409 , 0.23066047, 0.24973206,
       0.26910165, 0.22207716, 0.25862432, 0.23024742, 0.22372228,
       0.23422919, 0.24650474, 0.2856742 , 0.2793826 , 0.26312664,
       0.23928094, 0.26544127, 0.25757164, 0.2025107 , 0.20041946,
       0.28672475, 0.25480896, 0.22854814, 0.24869782, 0.23096187,
       0.2813165 , 0.27034017, 0.2569795 , 0.21727042, 0.28055146,
       0.27281886, 0.25968137, 0.27898005, 0.22259375, 0.2377495 ,
       0.22099793, 0.2507727 , 0.24659538, 0.22101505, 0.22632588,
       0.25414217, 0.24685644, 0.23780593, 0.25982112, 0.25838885,
       0.22764745, 0.24200216, 0.18154444, 0.27551985, 0.2583293 ,
       0.24426825, 0.2732193 , 0.23977351, 0.23207869, 0.2239526 ,
       0.21523343, 0.25460303, 0.27394077, 0.23523484, 0.23723707,
       0.222791  , 0.18889476, 0.21227987, 0.2113625 , 0.2202905 ,
       0.2670882 , 0.26453573, 0.26543015, 0.28290066, 0.27036318,
       0.28161994, 0.23983459, 0.2399843 , 0.2538837 , 0.28617197],      dtype=float32)
                },
                'post': {
                    'b': Array([ 0.00227712,  0.00410922,  0.0058516 , ...,  0.00019771,
       -0.00454297,  0.00112776], dtype=float32),
                    'dora_m': Array([[[0.44481152, 0.42009625, 0.42199472, ..., 0.4160813 ,
         0.42286402, 0.42347634],
        [0.39237085, 0.42473966, 0.40693057, ..., 0.4075258 ,
         0.42757353, 0.41165352],
        [0.43527588, 0.41222858, 0.41618323, ..., 0.39936894,
         0.40906224, 0.38682538],
        ...,
        [0.37274486, 0.36327165, 0.3754796 , ..., 0.33363336,
         0.39488873, 0.38433388],
        [0.4378884 , 0.43301976, 0.41809502, ..., 0.41065952,
         0.4082298 , 0.44112903],
        [0.40753263, 0.4244671 , 0.42001125, ..., 0.42006347,
         0.4084677 , 0.4302399 ]]], dtype=float32),
                    'lora_a': Array([[-0.10849015],
       [ 0.29455188],
       [-0.26038763],
       ...,
       [-0.79826427],
       [ 0.73765177],
       [-0.08401327]], dtype=float32),
                    'lora_b': Array([[ 1.4652635e-05],
       [ 3.9353940e-06],
       [-2.4826118e-06],
       ...,
       [ 1.3200833e-05],
       [ 1.1052766e-05],
       [ 1.3657961e-05]], dtype=float32),
                    'w': Array([[[ 1.60219464e-02, -6.32830570e-03, -4.34190384e-04, ...,
         -2.05251835e-02,  5.95892640e-03, -4.76127584e-03],
        [-9.98676405e-05, -7.33074406e-03, -1.26863532e-02, ...,
          7.18150195e-03, -1.75036013e-03,  1.55169265e-02],
        [-5.81251038e-03, -5.87717863e-03,  2.19022054e-02, ...,
         -5.58457756e-03,  1.16657428e-02, -1.81494025e-03],
        ...,
        [-7.13085895e-03,  2.03952119e-02, -9.58707742e-03, ...,
         -1.69843007e-02,  9.77954641e-03, -2.47077830e-03],
        [-1.87334344e-02, -2.76322174e-03,  2.28968691e-02, ...,
         -3.18435661e-04,  1.49174239e-02, -2.33985633e-02],
        [-8.63831770e-03, -4.08300431e-03,  2.85220704e-05, ...,
          2.67661619e-03, -9.26318858e-03, -3.84686440e-02]],

...,

[[-5.88139240e-03, -2.03647055e-02, -1.63833331e-02, ...,
         -1.43157914e-02, -2.27124058e-02, -1.14719691e-02],
        [ 1.83314341e-03, -2.54484490e-02, -8.61003343e-03, ...,
          4.26080049e-04, -1.36934174e-02,  1.51861394e-02],
        [-1.15379028e-03, -3.19069833e-03, -2.13307329e-03, ...,
         -2.33514644e-02,  2.00367980e-02, -6.51050312e-03],
        ...,
        [ 7.55283562e-03, -9.96225234e-03,  1.34144872e-02, ...,
          6.54450199e-03, -3.36529757e-03,  5.97001240e-03],
        [ 3.44229583e-03,  9.81338508e-03, -1.04225297e-02, ...,
         -9.26869828e-03, -3.25994082e-02, -1.25933774e-02],
        [ 1.09423827e-02, -1.90035310e-02,  6.49145385e-03, ...,
          3.46495374e-03,  9.23398882e-03,  1.87372081e-02]]],      dtype=float32)
                },
                'query': {
                    'b': Array([[ 0.00630543, -0.01040035, -0.00061541, ...,  0.00912199,
        -0.00037946, -0.00610306],
       [ 0.01307749, -0.0054321 ,  0.00767672, ...,  0.0043904 ,
         0.00037232,  0.01104123],
       [ 0.01931557,  0.00255344, -0.01728779, ..., -0.00125953,
         0.00231459,  0.02259091],
       ...,
       [ 0.00335209,  0.01204891, -0.02524816, ..., -0.00921736,
        -0.00198141,  0.01285304],
       [ 0.03694161, -0.01562951, -0.00046486, ...,  0.01202064,
         0.02256216,  0.0066755 ],
       [-0.01016504,  0.01377161,  0.01127281, ..., -0.01075078,
         0.01636219, -0.00281313]], dtype=float32),
                    'dora_m': Array([[[0.6983665 , 0.7122263 , 0.8625845 , ..., 0.79464024,
         0.7668895 , 0.85161906],
        [0.6864689 , 0.73798764, 0.7123014 , ..., 0.76975167,
         0.7144764 , 0.80368376],
        [0.88515645, 0.7195766 , 0.8433217 , ..., 0.8656572 ,
         0.7809146 , 0.909042  ],
        ...,
        [0.54963535, 0.49565613, 0.59163284, ..., 0.56806624,
         0.57028425, 0.6666808 ],
        [0.76513404, 0.7269381 , 0.80446917, ..., 0.7865502 ,
         0.6813359 , 0.61398524],
        [0.6441152 , 0.6876091 , 0.7174332 , ..., 0.68692064,
         0.60526115, 0.7539159 ]]], dtype=float32),
                    'lora_a': Array([[-0.10848287],
       [ 0.29454795],
       [-0.2603884 ],
       ...,
       [-0.7982765 ],
       [ 0.7376153 ],
       [-0.08403523]], dtype=float32),
                    'lora_b': Array([[-7.4309041e-06],
       [-7.0362939e-06],
       [-7.9890797e-06],
       ...,
       [ 2.2137467e-05],
       [ 1.6183387e-05],
       [-7.6748711e-06]], dtype=float32),
                    'w': Array([[[ 0.00683608,  0.01803902, -0.0127693 , ..., -0.00610208,
          0.05388   ,  0.00311598],
        [-0.01006073, -0.00214602,  0.01311889, ..., -0.02653091,
          0.00350361,  0.00938338],
        [-0.02756437, -0.03875   , -0.01696985, ..., -0.00325368,
         -0.04992706,  0.03636185],
        ...,
        [ 0.04123638,  0.03804113, -0.03609247, ..., -0.0377653 ,
         -0.01802078, -0.00413028],
        [ 0.01214722,  0.01488034,  0.02144075, ...,  0.01663101,
         -0.00706379, -0.00448915],
        [-0.01851939, -0.00234487, -0.02068215, ..., -0.00288961,
          0.04119675,  0.02511111]],

...,

[[-0.02320945,  0.03080179, -0.01184121, ..., -0.02594195,
         -0.0131538 , -0.01832458],
        [-0.00467245,  0.03966678, -0.00394025, ...,  0.02598585,
          0.01235735,  0.02955044],
        [-0.03318202, -0.00447797,  0.00395965, ...,  0.02871543,
          0.0354715 , -0.03632242],
        ...,
        [ 0.02489721,  0.01103833, -0.00312272, ..., -0.00588333,
          0.02076965,  0.01153785],
        [ 0.01564988, -0.01428127, -0.01250215, ..., -0.01380367,
         -0.01107514, -0.04016251],
        [ 0.03295957, -0.00897606, -0.01479999, ...,  0.01106019,
         -0.01252355,  0.01575936]]], dtype=float32)
                },
                'value': {
                    'b': Array([[-1.6719506e-03,  3.1838890e-03, -3.4906643e-03, ...,
         2.1193527e-04, -1.3315614e-03, -2.9562660e-03],
       [-3.5157211e-03,  5.4036635e-03, -5.3099038e-06, ...,
        -1.5847052e-03,  1.8464231e-04,  7.0413254e-04],
       [-2.6758336e-03, -2.0398316e-03, -1.1476204e-03, ...,
        -5.6534848e-04, -1.3752164e-04, -2.1303194e-03],
       ...,
       [ 5.4894928e-03, -6.2388863e-04, -1.8957579e-03, ...,
         5.6731803e-03,  3.9430051e-03, -4.2474028e-03],
       [-2.0500964e-03,  1.6432669e-03, -4.0994561e-03, ...,
        -1.0819612e-03,  7.0421447e-05,  2.1111206e-03],
       [-5.5974408e-04,  3.3670918e-03,  1.4440225e-03, ...,
        -2.9739656e-04,  2.3998234e-03,  3.0667472e-03]], dtype=float32),
                    'dora_m': Array([[[0.48662817, 0.4828381 , 0.47895852, ..., 0.46218106,
         0.46520773, 0.4695908 ],
        [0.43447557, 0.47647515, 0.4507376 , ..., 0.45640275,
         0.4676275 , 0.4579889 ],
        [0.47736272, 0.44961777, 0.4614821 , ..., 0.461062  ,
         0.45644102, 0.45267883],
        ...,
        [0.4012654 , 0.44632003, 0.44956976, ..., 0.32980427,
         0.46751118, 0.4286886 ],
        [0.49121344, 0.49429753, 0.45360097, ..., 0.4693511 ,
         0.49022222, 0.47923982],
        [0.46886098, 0.47989082, 0.47667924, ..., 0.5002886 ,
         0.49479607, 0.4873353 ]]], dtype=float32),
                    'lora_a': Array([[-0.10849095],
       [ 0.29455352],
       [-0.26039168],
       ...,
       [-0.798279  ],
       [ 0.7376133 ],
       [-0.08404093]], dtype=float32),
                    'lora_b': Array([[-8.1582039e-06],
       [ 2.0970445e-05],
       [-2.1723383e-05],
       ...,
       [ 2.9871280e-06],
       [ 1.8161232e-05],
       [-1.6719114e-05]], dtype=float32),
                    'w': Array([[[-1.78365000e-02,  2.84314482e-03, -1.39748259e-02, ...,
          1.78501215e-02,  9.85812745e-04, -9.48785862e-04],
        [ 7.44687626e-03,  4.15558752e-04, -4.96728625e-03, ...,
          4.96215979e-03,  1.66751747e-03, -1.90461092e-02],
        [ 1.76583678e-02,  4.45447396e-03, -2.55572014e-02, ...,
          1.62308514e-02,  3.32947746e-02,  2.39235219e-02],
        ...,
        [ 2.78520212e-03,  1.96315497e-02,  6.07665209e-03, ...,
          1.57222468e-02,  3.04588443e-03, -4.29826090e-04],
        [-7.40560528e-04,  7.82942306e-03, -5.32508362e-03, ...,
         -2.35469788e-02, -3.91680887e-03,  1.02615301e-02],
        [-2.59020971e-03,  1.15150996e-02,  1.41894817e-03, ...,
          2.16003098e-02, -1.32964994e-03,  2.13793032e-02]],

...,

Words removed	335
Total words	1808
Words removed (%)	18.53

Words added	345
Total words	1818
Words added (%)	18.98