mirror of
https://github.com/ollama/ollama.git
synced 2026-11-08 02:16:57 -06:00
210 lines
55 KiB
JSON
210 lines
55 KiB
JSON
{
|
|
"source": "Qwen3RMSNorm/Qwen3RotaryEmbedding/apply_rotary_pos_emb: transformers v4.51.3; RoPE parameters: transformers v4.57.1; float32; rope_theta=500000; qk_norm=0.5",
|
|
"sample_tokens": [
|
|
0,
|
|
1,
|
|
7,
|
|
15
|
|
],
|
|
"sample_dims": [
|
|
0,
|
|
23,
|
|
30,
|
|
40,
|
|
63,
|
|
64,
|
|
87,
|
|
94,
|
|
104,
|
|
127
|
|
],
|
|
"cases": [
|
|
{
|
|
"name": "default",
|
|
"max_position_embeddings": 40960,
|
|
"rope_scaling": null,
|
|
"inv_freq": [1.0, 0.8146172165870667, 0.663601279258728, 0.5405809879302979, 0.44036662578582764, 0.3587302267551422, 0.2922278344631195, 0.2380538135766983, 0.193922758102417, 0.1579728126525879, 0.12868738174438477, 0.10483095049858093, 0.08539710193872452, 0.06956595182418823, 0.05666961893439293, 0.046164050698280334, 0.03760603070259094, 0.030634520575404167, 0.02495540864765644, 0.020329104736447334, 0.016560440883040428, 0.013490419834852219, 0.010989529080688953, 0.008952259086072445, 0.00729266507551074, 0.005940730683505535, 0.00483942124992609, 0.003942275885492563, 0.0032114461064338684, 0.0026160990819334984, 0.002131119603291154, 0.0017360467463731766, 0.001414213445968926, 0.001152042648755014, 0.0009384738514199853, 0.0007644969737157226, 0.0006227724370546639, 0.0005073211505077779, 0.0004132725589442998, 0.00033665893715806305, 0.00027424818836152554, 0.00022340727446135134, 0.00018199143232777715, 0.00014825335529167205, 0.00012076974235242233, 9.838111145654693e-05, 8.014294871827587e-05, 6.528582161990926e-05, 5.3182957344688475e-05, 4.332375465310179e-05, 3.529227615217678e-05, 2.8749695047736168e-05, 2.3419997887685895e-05, 1.907833393488545e-05, 1.5541540051344782e-05, 1.2660406355280429e-05, 1.0313385246263351e-05, 8.401460945606232e-06, 6.843975370429689e-06, 5.575220257014735e-06, 4.541670477919979e-06, 3.6997230381530244e-06, 3.01385807688348e-06, 2.4551407022954663e-06],
|
|
"attention_factor": 1.0,
|
|
"output_samples": [-1.125, 0.3125, 0.75, -0.9375, 0.5, 0.5625, -0.3125, 0.125, 0.75, -0.125, -0.07698158919811249, -0.9204629063606262, -0.48296287655830383, 0.11051841825246811, -0.7329629063606262, -0.6704628467559814, 0.7355184555053711, -1.107962965965271, -0.48296287655830383, 0.9230184555053711, -0.5068065524101257, 0.6231921911239624, -0.8396487832069397, -0.3629710078239441, 0.33127108216285706, 0.39377105236053467, 0.20752035081386566, 0.5804018378257751, -0.8396487832069397, 0.3950203061103821, 0.5698629021644592, 0.09191276878118515, 0.4576781094074249, 0.24600763618946075, 0.2542138695716858, 0.3167138695716858, -0.446920245885849, -0.06380360573530197, 0.4576781094074249, -0.2809051275253296],
|
|
"key_samples": [-0.838058590888977, 0.23279406130313873, 0.558705747127533, -0.6983821988105774, 0.372470498085022, 0.4190292954444885, -0.23279406130313873, 0.0931176245212555, 0.558705747127533, -0.0931176245212555, 0.41826963424682617, -0.7249672412872314, -0.3820733428001404, 0.09608406573534012, -0.5758744478225708, -0.32559821009635925, 0.5694054961204529, -0.8646252155303955, -0.38388878107070923, 0.7198393940925598, -0.9235590696334839, 0.6312442421913147, -0.8465202450752258, -0.3439614772796631, 0.7898766398429871, 0.3083733916282654, 0.18800339102745056, 0.4811018109321594, -0.839911162853241, 0.29621919989585876, -0.6591131687164307, 0.09881791472434998, 0.3626976013183594, 0.8085233569145203, 0.18001241981983185, 0.2680182456970215, -0.39533478021621704, -0.07844901829957962, 0.363334059715271, -0.26999711990356445],
|
|
"long_key_samples": [-0.7013427019119263, -0.3039819300174713, 0.35973766446113586, 0.39532166719436646, 0.3787490427494049, -0.6213260889053345, -0.12641650438308716, 0.43750637769699097, -0.8022534847259521, -0.06288349628448486, 0.4580969214439392, 0.8028610944747925, 0.2806074321269989, 0.07588721811771393, -0.6318604946136475, 0.2666662335395813, 0.45300573110580444, -0.9026716947555542, 0.38838621973991394, 0.6712319850921631, -0.6305563449859619, -0.0669579803943634, -0.9541407823562622, 0.6678540110588074, 0.7635167241096497, -0.7419260144233704, -0.6552337408065796, -0.1940896213054657, 0.6145985126495361, 0.3587382435798645, -0.4944906234741211, -0.403141587972641, 0.32485759258270264, -0.8862977027893066, 0.2011280655860901, -0.5116082429885864, 0.059425801038742065, 0.17936371266841888, 0.014071017503738403, -0.25465747714042664],
|
|
"long_output_samples": [-1.125, 0.3125, 0.75, -0.9375, 0.5, 0.5625, -0.3125, 0.125, 0.75, -0.125, -0.0769815519452095, -0.9204629063606262, -0.4829629063606262, 0.1105184480547905, -0.7329629063606262, -0.6704629063606262, 0.7355184555053711, -1.107962965965271, -0.4829629063606262, 0.9230184555053711, -0.5068061351776123, 0.6232134103775024, -0.8396475911140442, -0.36297014355659485, 0.33121639490127563, 0.39371639490127563, 0.20752526819705963, 0.5804116725921631, -0.8396475911140442, 0.39502525329589844, 0.5698559880256653, 0.09191132336854935, 0.45768284797668457, 0.24600550532341003, 0.25421223044395447, 0.31671223044395447, -0.44692519307136536, -0.06380632519721985, 0.45768284797668457, -0.2809096872806549]
|
|
},
|
|
{
|
|
"name": "yarn4",
|
|
"max_position_embeddings": 131072,
|
|
"rope_scaling": {
|
|
"rope_type": "yarn",
|
|
"factor": 4.0,
|
|
"original_max_position_embeddings": 32768
|
|
},
|
|
"inv_freq": [1.0, 0.8146172165870667, 0.663601279258728, 0.5405809879302979, 0.44036662578582764, 0.3587302267551422, 0.2922278344631195, 0.2380538135766983, 0.193922758102417, 0.1579728126525879, 0.12868738174438477, 0.10483095049858093, 0.08539710193872452, 0.06956595182418823, 0.05666961893439293, 0.046164050698280334, 0.03760603070259094, 0.030634520575404167, 0.02495540864765644, 0.020329104736447334, 0.016560440883040428, 0.013490419834852219, 0.010989529080688953, 0.008952259086072445, 0.00729266507551074, 0.005693200044333935, 0.004436136223375797, 0.0034494914580136538, 0.0026762052439153194, 0.0020710784010589123, 0.0015983396442607045, 0.0012296997010707855, 0.0009428090415894985, 0.0007200266700237989, 0.000547443050891161, 0.00041410254198126495, 0.0003113861894235015, 0.00023252220125868917, 0.0001721968874335289, 0.00012624710507225245, 9.141606278717518e-05, 6.516046414617449e-05, 4.549785808194429e-05, 3.706333882291801e-05, 3.0192435588105582e-05, 2.4595277864136733e-05, 2.003573717956897e-05, 1.6321455404977314e-05, 1.3295739336172119e-05, 1.0830938663275447e-05, 8.823069038044196e-06, 7.187423761934042e-06, 5.854999471921474e-06, 4.7695834837213624e-06, 3.8853850128361955e-06, 3.1651015888201073e-06, 2.578346311565838e-06, 2.100365236401558e-06, 1.7109938426074223e-06, 1.3938050642536837e-06, 1.1354176194799948e-06, 9.249307595382561e-07, 7.5346451922087e-07, 6.137851755738666e-07],
|
|
"attention_factor": 1.138629436111989,
|
|
"output_samples": [-1.125, 0.3125, 0.75, -0.9375, 0.5, 0.5625, -0.3125, 0.125, 0.75, -0.125, -0.06660930067300797, -0.9326654672622681, -0.49516549706459045, 0.12089069932699203, -0.7451654672622681, -0.6826654672622681, 0.7458906769752502, -1.120165467262268, -0.49516549706459045, 0.9333906769752502, -0.5575152039527893, 0.7235569357872009, -0.9375719428062439, -0.38849642872810364, 0.49885261058807373, 0.5613526105880737, 0.21633166074752808, 0.623228132724762, -0.9375719428062439, 0.4038316607475281, 0.6961049437522888, 0.09358730167150497, 0.49438023567199707, 0.4224497079849243, 0.26966211199760437, 0.33216211199760437, -0.4962541162967682, -0.08124681562185287, 0.49438023567199707, -0.31806203722953796],
|
|
"key_samples": [-0.9542381763458252, 0.2650661766529083, 0.63615882396698, -0.7951985001564026, 0.4241058826446533, 0.4771190881729126, -0.2650661766529083, 0.10602647066116333, 0.63615882396698, -0.10602647066116333, 0.4762541353702545, -0.8254690766334534, -0.43556439876556396, 0.10932422429323196, -0.6557060480117798, -0.3707357347011566, 0.6483418345451355, -0.9842557907104492, -0.4371270537376404, 0.8196315169334412, -1.0515915155410767, 0.7187532186508179, -0.9618231654167175, -0.392868310213089, 0.899381160736084, 0.35112303495407104, 0.21406617760658264, 0.5513875484466553, -0.9558455348014832, 0.3372723460197449, -0.7504856586456299, 0.11251698434352875, 0.41225114464759827, 0.9217395782470703, 0.2049589604139328, 0.30517345666885376, -0.4501397907733917, -0.09262187033891678, 0.41117656230926514, -0.3074323236942291],
|
|
"long_key_samples": [-0.7985695600509644, -0.3461228013038635, -0.41708531975746155, 0.6941357851028442, 0.42615240812301636, -0.7074602246284485, -0.14394155144691467, 0.49191415309906006, -0.7451270222663879, -0.09747575968503952, 0.5216026306152344, 0.9141613245010376, 1.0683919191360474, -0.04453512281179428, -0.6720572113990784, 0.30363404750823975, 0.5158056616783142, 0.13043993711471558, 0.44838428497314453, 0.8062787652015686, -0.7179700136184692, -0.07624031603336334, 0.01948830485343933, 0.5278098583221436, 0.89241623878479, -0.8447787761688232, -0.7460684180259705, -1.1084909439086914, 0.888483464717865, 0.3552916944026947, -0.563041627407074, -0.4590288996696472, -0.13175401091575623, -0.9717712998390198, 0.21110032498836517, -0.5825321674346924, 0.06766396760940552, 0.4014606773853302, -0.27263641357421875, -0.30324819684028625],
|
|
"long_output_samples": [-1.125, 0.3125, 0.75, -0.9375, 0.5, 0.5625, -0.3125, 0.125, 0.75, -0.125, -0.06660927832126617, -0.9326655864715576, -0.4951655864715576, 0.12089072912931442, -0.7451655864715576, -0.6826655864715576, 0.745890736579895, -1.1201655864715576, -0.4951655864715576, 0.9333907961845398, -0.5575114488601685, 0.7235738039016724, -0.9375671744346619, -0.38849249482154846, 0.49878236651420593, 0.5612823367118835, 0.21633726358413696, 0.6232365965843201, -0.9375671744346619, 0.40383729338645935, 0.696094810962677, 0.09358564019203186, 0.4943827986717224, 0.4224432110786438, 0.2696602940559387, 0.33216026425361633, -0.4962575137615204, -0.08124911040067673, 0.4943827986717224, -0.31806522607803345]
|
|
},
|
|
{
|
|
"name": "unextended_limit",
|
|
"max_position_embeddings": 40960,
|
|
"rope_scaling": {
|
|
"rope_type": "yarn",
|
|
"factor": 4.0,
|
|
"original_max_position_embeddings": 32768
|
|
},
|
|
"inv_freq": [1.0, 0.8146172165870667, 0.663601279258728, 0.5405809879302979, 0.44036662578582764, 0.3587302267551422, 0.2922278344631195, 0.2380538135766983, 0.193922758102417, 0.1579728126525879, 0.12868738174438477, 0.10483095049858093, 0.08539710193872452, 0.06956595182418823, 0.05666961893439293, 0.046164050698280334, 0.03760603070259094, 0.030634520575404167, 0.02495540864765644, 0.020329104736447334, 0.016560440883040428, 0.013490419834852219, 0.010989529080688953, 0.008952259086072445, 0.00729266507551074, 0.005693200044333935, 0.004436136223375797, 0.0034494914580136538, 0.0026762052439153194, 0.0020710784010589123, 0.0015983396442607045, 0.0012296997010707855, 0.0009428090415894985, 0.0007200266700237989, 0.000547443050891161, 0.00041410254198126495, 0.0003113861894235015, 0.00023252220125868917, 0.0001721968874335289, 0.00012624710507225245, 9.141606278717518e-05, 6.516046414617449e-05, 4.549785808194429e-05, 3.706333882291801e-05, 3.0192435588105582e-05, 2.4595277864136733e-05, 2.003573717956897e-05, 1.6321455404977314e-05, 1.3295739336172119e-05, 1.0830938663275447e-05, 8.823069038044196e-06, 7.187423761934042e-06, 5.854999471921474e-06, 4.7695834837213624e-06, 3.8853850128361955e-06, 3.1651015888201073e-06, 2.578346311565838e-06, 2.100365236401558e-06, 1.7109938426074223e-06, 1.3938050642536837e-06, 1.1354176194799948e-06, 9.249307595382561e-07, 7.5346451922087e-07, 6.137851755738666e-07],
|
|
"attention_factor": 1.138629436111989,
|
|
"output_samples": [-1.125, 0.3125, 0.75, -0.9375, 0.5, 0.5625, -0.3125, 0.125, 0.75, -0.125, -0.06660930067300797, -0.9326654672622681, -0.49516549706459045, 0.12089069932699203, -0.7451654672622681, -0.6826654672622681, 0.7458906769752502, -1.120165467262268, -0.49516549706459045, 0.9333906769752502, -0.5575152039527893, 0.7235569357872009, -0.9375719428062439, -0.38849642872810364, 0.49885261058807373, 0.5613526105880737, 0.21633166074752808, 0.623228132724762, -0.9375719428062439, 0.4038316607475281, 0.6961049437522888, 0.09358730167150497, 0.49438023567199707, 0.4224497079849243, 0.26966211199760437, 0.33216211199760437, -0.4962541162967682, -0.08124681562185287, 0.49438023567199707, -0.31806203722953796],
|
|
"key_samples": [-0.9542381763458252, 0.2650661766529083, 0.63615882396698, -0.7951985001564026, 0.4241058826446533, 0.4771190881729126, -0.2650661766529083, 0.10602647066116333, 0.63615882396698, -0.10602647066116333, 0.4762541353702545, -0.8254690766334534, -0.43556439876556396, 0.10932422429323196, -0.6557060480117798, -0.3707357347011566, 0.6483418345451355, -0.9842557907104492, -0.4371270537376404, 0.8196315169334412, -1.0515915155410767, 0.7187532186508179, -0.9618231654167175, -0.392868310213089, 0.899381160736084, 0.35112303495407104, 0.21406617760658264, 0.5513875484466553, -0.9558455348014832, 0.3372723460197449, -0.7504856586456299, 0.11251698434352875, 0.41225114464759827, 0.9217395782470703, 0.2049589604139328, 0.30517345666885376, -0.4501397907733917, -0.09262187033891678, 0.41117656230926514, -0.3074323236942291],
|
|
"long_key_samples": [-0.7985695600509644, -0.3461228013038635, -0.41708531975746155, 0.6941357851028442, 0.42615240812301636, -0.7074602246284485, -0.14394155144691467, 0.49191415309906006, -0.7451270222663879, -0.09747575968503952, 0.5216026306152344, 0.9141613245010376, 1.0683919191360474, -0.04453512281179428, -0.6720572113990784, 0.30363404750823975, 0.5158056616783142, 0.13043993711471558, 0.44838428497314453, 0.8062787652015686, -0.7179700136184692, -0.07624031603336334, 0.01948830485343933, 0.5278098583221436, 0.89241623878479, -0.8447787761688232, -0.7460684180259705, -1.1084909439086914, 0.888483464717865, 0.3552916944026947, -0.563041627407074, -0.4590288996696472, -0.13175401091575623, -0.9717712998390198, 0.21110032498836517, -0.5825321674346924, 0.06766396760940552, 0.4014606773853302, -0.27263641357421875, -0.30324819684028625],
|
|
"long_output_samples": [-1.125, 0.3125, 0.75, -0.9375, 0.5, 0.5625, -0.3125, 0.125, 0.75, -0.125, -0.06660927832126617, -0.9326655864715576, -0.4951655864715576, 0.12089072912931442, -0.7451655864715576, -0.6826655864715576, 0.745890736579895, -1.1201655864715576, -0.4951655864715576, 0.9333907961845398, -0.5575114488601685, 0.7235738039016724, -0.9375671744346619, -0.38849249482154846, 0.49878236651420593, 0.5612823367118835, 0.21633726358413696, 0.6232365965843201, -0.9375671744346619, 0.40383729338645935, 0.696094810962677, 0.09358564019203186, 0.4943827986717224, 0.4224432110786438, 0.2696602940559387, 0.33216026425361633, -0.4962575137615204, -0.08124911040067673, 0.4943827986717224, -0.31806522607803345]
|
|
},
|
|
{
|
|
"name": "implicit_original",
|
|
"max_position_embeddings": 32768,
|
|
"rope_scaling": {
|
|
"rope_type": "yarn",
|
|
"factor": 4.0
|
|
},
|
|
"inv_freq": [1.0, 0.8146172165870667, 0.663601279258728, 0.5405809879302979, 0.44036662578582764, 0.3587302267551422, 0.2922278344631195, 0.2380538135766983, 0.193922758102417, 0.1579728126525879, 0.12868738174438477, 0.10483095049858093, 0.08539710193872452, 0.06956595182418823, 0.05666961893439293, 0.046164050698280334, 0.03760603070259094, 0.030634520575404167, 0.02495540864765644, 0.020329104736447334, 0.016560440883040428, 0.013490419834852219, 0.010989529080688953, 0.008952259086072445, 0.00729266507551074, 0.005693200044333935, 0.004436136223375797, 0.0034494914580136538, 0.0026762052439153194, 0.0020710784010589123, 0.0015983396442607045, 0.0012296997010707855, 0.0009428090415894985, 0.0007200266700237989, 0.000547443050891161, 0.00041410254198126495, 0.0003113861894235015, 0.00023252220125868917, 0.0001721968874335289, 0.00012624710507225245, 9.141606278717518e-05, 6.516046414617449e-05, 4.549785808194429e-05, 3.706333882291801e-05, 3.0192435588105582e-05, 2.4595277864136733e-05, 2.003573717956897e-05, 1.6321455404977314e-05, 1.3295739336172119e-05, 1.0830938663275447e-05, 8.823069038044196e-06, 7.187423761934042e-06, 5.854999471921474e-06, 4.7695834837213624e-06, 3.8853850128361955e-06, 3.1651015888201073e-06, 2.578346311565838e-06, 2.100365236401558e-06, 1.7109938426074223e-06, 1.3938050642536837e-06, 1.1354176194799948e-06, 9.249307595382561e-07, 7.5346451922087e-07, 6.137851755738666e-07],
|
|
"attention_factor": 1.138629436111989,
|
|
"output_samples": [-1.125, 0.3125, 0.75, -0.9375, 0.5, 0.5625, -0.3125, 0.125, 0.75, -0.125, -0.06660930067300797, -0.9326654672622681, -0.49516549706459045, 0.12089069932699203, -0.7451654672622681, -0.6826654672622681, 0.7458906769752502, -1.120165467262268, -0.49516549706459045, 0.9333906769752502, -0.5575152039527893, 0.7235569357872009, -0.9375719428062439, -0.38849642872810364, 0.49885261058807373, 0.5613526105880737, 0.21633166074752808, 0.623228132724762, -0.9375719428062439, 0.4038316607475281, 0.6961049437522888, 0.09358730167150497, 0.49438023567199707, 0.4224497079849243, 0.26966211199760437, 0.33216211199760437, -0.4962541162967682, -0.08124681562185287, 0.49438023567199707, -0.31806203722953796],
|
|
"key_samples": [-0.9542381763458252, 0.2650661766529083, 0.63615882396698, -0.7951985001564026, 0.4241058826446533, 0.4771190881729126, -0.2650661766529083, 0.10602647066116333, 0.63615882396698, -0.10602647066116333, 0.4762541353702545, -0.8254690766334534, -0.43556439876556396, 0.10932422429323196, -0.6557060480117798, -0.3707357347011566, 0.6483418345451355, -0.9842557907104492, -0.4371270537376404, 0.8196315169334412, -1.0515915155410767, 0.7187532186508179, -0.9618231654167175, -0.392868310213089, 0.899381160736084, 0.35112303495407104, 0.21406617760658264, 0.5513875484466553, -0.9558455348014832, 0.3372723460197449, -0.7504856586456299, 0.11251698434352875, 0.41225114464759827, 0.9217395782470703, 0.2049589604139328, 0.30517345666885376, -0.4501397907733917, -0.09262187033891678, 0.41117656230926514, -0.3074323236942291],
|
|
"long_key_samples": [-0.7985695600509644, -0.3461228013038635, -0.41708531975746155, 0.6941357851028442, 0.42615240812301636, -0.7074602246284485, -0.14394155144691467, 0.49191415309906006, -0.7451270222663879, -0.09747575968503952, 0.5216026306152344, 0.9141613245010376, 1.0683919191360474, -0.04453512281179428, -0.6720572113990784, 0.30363404750823975, 0.5158056616783142, 0.13043993711471558, 0.44838428497314453, 0.8062787652015686, -0.7179700136184692, -0.07624031603336334, 0.01948830485343933, 0.5278098583221436, 0.89241623878479, -0.8447787761688232, -0.7460684180259705, -1.1084909439086914, 0.888483464717865, 0.3552916944026947, -0.563041627407074, -0.4590288996696472, -0.13175401091575623, -0.9717712998390198, 0.21110032498836517, -0.5825321674346924, 0.06766396760940552, 0.4014606773853302, -0.27263641357421875, -0.30324819684028625],
|
|
"long_output_samples": [-1.125, 0.3125, 0.75, -0.9375, 0.5, 0.5625, -0.3125, 0.125, 0.75, -0.125, -0.06660927832126617, -0.9326655864715576, -0.4951655864715576, 0.12089072912931442, -0.7451655864715576, -0.6826655864715576, 0.745890736579895, -1.1201655864715576, -0.4951655864715576, 0.9333907961845398, -0.5575114488601685, 0.7235738039016724, -0.9375671744346619, -0.38849249482154846, 0.49878236651420593, 0.5612823367118835, 0.21633726358413696, 0.6232365965843201, -0.9375671744346619, 0.40383729338645935, 0.696094810962677, 0.09358564019203186, 0.4943827986717224, 0.4224432110786438, 0.2696602940559387, 0.33216026425361633, -0.4962575137615204, -0.08124911040067673, 0.4943827986717224, -0.31806522607803345]
|
|
},
|
|
{
|
|
"name": "beta",
|
|
"max_position_embeddings": 131072,
|
|
"rope_scaling": {
|
|
"rope_type": "yarn",
|
|
"factor": 4.0,
|
|
"original_max_position_embeddings": 32768,
|
|
"beta_fast": 16.0,
|
|
"beta_slow": 2.0
|
|
},
|
|
"inv_freq": [1.0, 0.8146172165870667, 0.663601279258728, 0.5405809879302979, 0.44036662578582764, 0.3587302267551422, 0.2922278344631195, 0.2380538135766983, 0.193922758102417, 0.1579728126525879, 0.12868738174438477, 0.10483095049858093, 0.08539710193872452, 0.06956595182418823, 0.05666961893439293, 0.046164050698280334, 0.03760603070259094, 0.030634520575404167, 0.02495540864765644, 0.020329104736447334, 0.016560440883040428, 0.013490419834852219, 0.010989529080688953, 0.008952259086072445, 0.00729266507551074, 0.005940730683505535, 0.00483942124992609, 0.003942275885492563, 0.0032114461064338684, 0.002437728689983487, 0.001840512384660542, 0.0013809462543576956, 0.0010285188909620047, 0.0007593007758259773, 0.0005545527092181146, 0.0003996234154328704, 0.0002830783778335899, 0.00019601044186856598, 0.00013149580627214164, 8.416473428951576e-05, 6.856204709038138e-05, 5.5851818615337834e-05, 4.549785808194429e-05, 3.706333882291801e-05, 3.0192435588105582e-05, 2.4595277864136733e-05, 2.003573717956897e-05, 1.6321455404977314e-05, 1.3295739336172119e-05, 1.0830938663275447e-05, 8.823069038044196e-06, 7.187423761934042e-06, 5.854999471921474e-06, 4.7695834837213624e-06, 3.8853850128361955e-06, 3.1651015888201073e-06, 2.578346311565838e-06, 2.100365236401558e-06, 1.7109938426074223e-06, 1.3938050642536837e-06, 1.1354176194799948e-06, 9.249307595382561e-07, 7.5346451922087e-07, 6.137851755738666e-07],
|
|
"attention_factor": 1.138629436111989,
|
|
"output_samples": [-1.125, 0.3125, 0.75, -0.9375, 0.5, 0.5625, -0.3125, 0.125, 0.75, -0.125, -0.06660912930965424, -0.9326657652854919, -0.49516573548316956, 0.12089087069034576, -0.7451657652854919, -0.6826657652854919, 0.7458909153938293, -1.1201657056808472, -0.49516573548316956, 0.9333909153938293, -0.5574978590011597, 0.7235206961631775, -0.9375842809677124, -0.38847315311431885, 0.4987613558769226, 0.5612613558769226, 0.21635834872722626, 0.6232239007949829, -0.9375842809677124, 0.40385836362838745, 0.6960664987564087, 0.09360380470752716, 0.4943825304508209, 0.4224020540714264, 0.26967567205429077, 0.3321756422519684, -0.4962383806705475, -0.08122789114713669, 0.4943825304508209, -0.3180432915687561],
|
|
"key_samples": [-0.9542381763458252, 0.2650661766529083, 0.63615882396698, -0.7951985001564026, 0.4241058826446533, 0.4771190881729126, -0.2650661766529083, 0.10602647066116333, 0.63615882396698, -0.10602647066116333, 0.4762541353702545, -0.8254690766334534, -0.4353260397911072, 0.10931423306465149, -0.6557060480117798, -0.3707357347011566, 0.6483418345451355, -0.9843612313270569, -0.43712955713272095, 0.8196315169334412, -1.0515915155410767, 0.7187532186508179, -0.9627565741539001, -0.39302119612693787, 0.899381160736084, 0.35112303495407104, 0.21406617760658264, 0.5497563481330872, -0.9557827115058899, 0.3372723460197449, -0.7504856586456299, 0.11251698434352875, 0.4125848710536957, 0.9218804836273193, 0.2049589604139328, 0.30517345666885376, -0.4501397907733917, -0.09112371504306793, 0.410860538482666, -0.3074323236942291],
|
|
"long_key_samples": [-0.7985695600509644, -0.3461228013038635, -0.4563106596469879, 0.0011241734027862549, 0.42615240812301636, -0.7074602246284485, -0.14394155144691467, -0.4557633399963379, -1.01835036277771, -0.09747575968503952, 0.5216026306152344, 0.9141613245010376, -0.2172745168209076, 0.2726568877696991, -0.6720572113990784, 0.30363404750823975, 0.5158056616783142, 1.0541667938232422, 0.3587339520454407, 0.8062787652015686, -0.7179700136184692, -0.07624031603336334, 1.1030337810516357, 0.991538405418396, 0.89241623878479, -0.8447787761688232, -0.7460684180259705, 0.11157238483428955, 0.2912694215774536, 0.3552916944026947, -0.563041627407074, -0.4590288996696472, -0.38879817724227905, -0.8972331881523132, 0.21110032498836517, -0.5825321674346924, 0.06766396760940552, -0.165426105260849, 0.46221497654914856, -0.30324819684028625],
|
|
"long_output_samples": [-1.125, 0.3125, 0.75, -0.9375, 0.5, 0.5625, -0.3125, 0.125, 0.75, -0.125, -0.06660910695791245, -0.9326657652854919, -0.49516576528549194, 0.12089089304208755, -0.7451657652854919, -0.6826657652854919, 0.7458909153938293, -1.1201657056808472, -0.49516576528549194, 0.9333909153938293, -0.557494044303894, 0.7235375642776489, -0.9375793933868408, -0.3884691596031189, 0.49869033694267273, 0.5611903667449951, 0.2163640558719635, 0.6232324242591858, -0.9375793933868408, 0.4038640260696411, 0.6960564851760864, 0.09360211342573166, 0.49438509345054626, 0.4223956763744354, 0.26967382431030273, 0.3321738541126251, -0.49624183773994446, -0.08123022317886353, 0.49438509345054626, -0.31804654002189636]
|
|
},
|
|
{
|
|
"name": "fractional_range",
|
|
"max_position_embeddings": 131072,
|
|
"rope_scaling": {
|
|
"rope_type": "yarn",
|
|
"factor": 4.0,
|
|
"original_max_position_embeddings": 32768,
|
|
"truncate": false
|
|
},
|
|
"inv_freq": [1.0, 0.8146172165870667, 0.663601279258728, 0.5405809879302979, 0.44036662578582764, 0.3587302267551422, 0.2922278344631195, 0.2380538135766983, 0.193922758102417, 0.1579728126525879, 0.12868738174438477, 0.10483095049858093, 0.08539710193872452, 0.06956595182418823, 0.05666961893439293, 0.046164050698280334, 0.03760603070259094, 0.030634520575404167, 0.02495540864765644, 0.020329104736447334, 0.016560440883040428, 0.013490419834852219, 0.010989529080688953, 0.008952259086072445, 0.00729266507551074, 0.005899169482290745, 0.0045908354222774506, 0.0035648515913635492, 0.0027614948339760303, 0.002133482601493597, 0.0016434122808277607, 0.0012617219472303987, 0.0009650704450905323, 0.00073504593456164, 0.0005571402725763619, 0.00041993462946265936, 0.0003144530928693712, 0.00023364860680885613, 0.0001719969295663759, 0.00012517381401266903, 8.980011625681072e-05, 6.323994603008032e-05, 4.549785808194429e-05, 3.706333882291801e-05, 3.0192435588105582e-05, 2.4595277864136733e-05, 2.003573717956897e-05, 1.6321455404977314e-05, 1.3295739336172119e-05, 1.0830938663275447e-05, 8.823069038044196e-06, 7.187423761934042e-06, 5.854999471921474e-06, 4.7695834837213624e-06, 3.8853850128361955e-06, 3.1651015888201073e-06, 2.578346311565838e-06, 2.100365236401558e-06, 1.7109938426074223e-06, 1.3938050642536837e-06, 1.1354176194799948e-06, 9.249307595382561e-07, 7.5346451922087e-07, 6.137851755738666e-07],
|
|
"attention_factor": 1.138629436111989,
|
|
"output_samples": [-1.125, 0.3125, 0.75, -0.9375, 0.5, 0.5625, -0.3125, 0.125, 0.75, -0.125, -0.06660925596952438, -0.9326655864715576, -0.49516561627388, 0.12089075148105621, -0.7451655864715576, -0.6826656460762024, 0.745890736579895, -1.1201655864715576, -0.49516561627388, 0.9333907961845398, -0.55750972032547, 0.7235428690910339, -0.9375758171081543, -0.3884887397289276, 0.49882370233535767, 0.5613237023353577, 0.21634051203727722, 0.623227059841156, -0.9375758171081543, 0.4038405120372772, 0.6960883736610413, 0.09359432756900787, 0.49438223242759705, 0.42242956161499023, 0.2696681320667267, 0.3321681320667267, -0.4962489902973175, -0.08124002814292908, 0.49438223242759705, -0.3180558681488037],
|
|
"key_samples": [-0.9542381763458252, 0.2650661766529083, 0.63615882396698, -0.7951985001564026, 0.4241058826446533, 0.4771190881729126, -0.2650661766529083, 0.10602647066116333, 0.63615882396698, -0.10602647066116333, 0.4762541353702545, -0.8254690766334534, -0.4355200529098511, 0.10932351648807526, -0.6557060480117798, -0.3707357347011566, 0.6483418345451355, -0.9842754006385803, -0.4371272325515747, 0.8196315169334412, -1.0515915155410767, 0.7187532186508179, -0.9619970917701721, -0.39287909865379333, 0.899381160736084, 0.35112303495407104, 0.21406617760658264, 0.5510841012001038, -0.9558411240577698, 0.3372723460197449, -0.7504856586456299, 0.11251698434352875, 0.41231366991996765, 0.921749472618103, 0.2049589604139328, 0.30517345666885376, -0.4501397907733917, -0.09234312921762466, 0.41115421056747437, -0.3074323236942291],
|
|
"long_key_samples": [-0.7985695600509644, -0.3461228013038635, -0.5288376808166504, 0.6537258625030518, 0.42615240812301636, -0.7074602246284485, -0.14394155144691467, -0.3691481947898865, -0.780820906162262, -0.09747575968503952, 0.5216026306152344, 0.9141613245010376, -0.0297870934009552, -0.020740605890750885, -0.6720572113990784, 0.30363404750823975, 0.5158056616783142, 1.0759128332138062, 0.4501129388809204, 0.8062787652015686, -0.7179700136184692, -0.07624031603336334, 1.1054654121398926, 0.5741041898727417, 0.89241623878479, -0.8447787761688232, -0.7460684180259705, -0.08413350582122803, 0.8592965602874756, 0.3552916944026947, -0.563041627407074, -0.4590288996696472, -0.41197827458381653, -0.9848442077636719, 0.21110032498836517, -0.5825321674346924, 0.06766396760940552, -0.09382805973291397, -0.22079803049564362, -0.30324819684028625],
|
|
"long_output_samples": [-1.125, 0.3125, 0.75, -0.9375, 0.5, 0.5625, -0.3125, 0.125, 0.75, -0.125, -0.06660923361778259, -0.9326655864715576, -0.49516561627388, 0.12089076638221741, -0.7451655864715576, -0.6826656460762024, 0.745890736579895, -1.1201655864715576, -0.49516561627388, 0.9333907961845398, -0.5575059652328491, 0.7235600352287292, -0.937571108341217, -0.3884848356246948, 0.4987529516220093, 0.5612529516220093, 0.21634620428085327, 0.6232356429100037, -0.937571108341217, 0.4038461744785309, 0.6960783004760742, 0.09359262138605118, 0.4943847358226776, 0.4224233329296112, 0.26966628432273865, 0.33216622471809387, -0.4962523281574249, -0.08124234527349472, 0.4943847358226776, -0.3180590569972992]
|
|
},
|
|
{
|
|
"name": "mscale",
|
|
"max_position_embeddings": 131072,
|
|
"rope_scaling": {
|
|
"rope_type": "yarn",
|
|
"factor": 4.0,
|
|
"original_max_position_embeddings": 32768,
|
|
"mscale": 2.0,
|
|
"mscale_all_dim": 1.0
|
|
},
|
|
"inv_freq": [1.0, 0.8146172165870667, 0.663601279258728, 0.5405809879302979, 0.44036662578582764, 0.3587302267551422, 0.2922278344631195, 0.2380538135766983, 0.193922758102417, 0.1579728126525879, 0.12868738174438477, 0.10483095049858093, 0.08539710193872452, 0.06956595182418823, 0.05666961893439293, 0.046164050698280334, 0.03760603070259094, 0.030634520575404167, 0.02495540864765644, 0.020329104736447334, 0.016560440883040428, 0.013490419834852219, 0.010989529080688953, 0.008952259086072445, 0.00729266507551074, 0.005693200044333935, 0.004436136223375797, 0.0034494914580136538, 0.0026762052439153194, 0.0020710784010589123, 0.0015983396442607045, 0.0012296997010707855, 0.0009428090415894985, 0.0007200266700237989, 0.000547443050891161, 0.00041410254198126495, 0.0003113861894235015, 0.00023252220125868917, 0.0001721968874335289, 0.00012624710507225245, 9.141606278717518e-05, 6.516046414617449e-05, 4.549785808194429e-05, 3.706333882291801e-05, 3.0192435588105582e-05, 2.4595277864136733e-05, 2.003573717956897e-05, 1.6321455404977314e-05, 1.3295739336172119e-05, 1.0830938663275447e-05, 8.823069038044196e-06, 7.187423761934042e-06, 5.854999471921474e-06, 4.7695834837213624e-06, 3.8853850128361955e-06, 3.1651015888201073e-06, 2.578346311565838e-06, 2.100365236401558e-06, 1.7109938426074223e-06, 1.3938050642536837e-06, 1.1354176194799948e-06, 9.249307595382561e-07, 7.5346451922087e-07, 6.137851755738666e-07],
|
|
"attention_factor": 1.121751143713058,
|
|
"output_samples": [-1.125, 0.3125, 0.75, -0.9375, 0.5, 0.5625, -0.3125, 0.125, 0.75, -0.125, -0.06733520328998566, -0.9318115711212158, -0.49431154131889343, 0.12016480416059494, -0.7443115711212158, -0.681811511516571, 0.7451648116111755, -1.1193115711212158, -0.49431154131889343, 0.9326648116111755, -0.552700400352478, 0.7137541770935059, -0.9278588891029358, -0.3859380781650543, 0.4791067838668823, 0.5416067838668823, 0.21603882312774658, 0.61970454454422, -0.9278588891029358, 0.4035387933254242, 0.6819217205047607, 0.09392955899238586, 0.49122869968414307, 0.39964208006858826, 0.26871457695961, 0.3312146067619324, -0.4914075434207916, -0.07923943549394608, 0.49122869968414307, -0.3143288195133209],
|
|
"key_samples": [-0.9400932192802429, 0.2611370086669922, 0.6267288327217102, -0.7834110856056213, 0.41781923174858093, 0.47004660964012146, -0.2611370086669922, 0.10445480793714523, 0.6267288327217102, -0.10445480793714523, 0.4691944420337677, -0.8132328987121582, -0.4291079044342041, 0.10770367085933685, -0.6459863185882568, -0.3652401864528656, 0.6387313008308411, -0.9696658253669739, -0.4306473731994629, 0.8074818849563599, -1.0360034704208374, 0.7080988883972168, -0.9475657939910889, -0.3870446979999542, 0.8860493302345276, 0.3459181785583496, 0.21089302003383636, 0.5432142019271851, -0.9416767954826355, 0.33227285742759705, -0.7393609881401062, 0.11084911227226257, 0.40614017844200134, 0.9080763459205627, 0.2019207775592804, 0.3006497919559479, -0.4434671998023987, -0.09124889969825745, 0.40508154034614563, -0.3028751611709595],
|
|
"long_key_samples": [-0.7867320775985718, -0.3409920930862427, -0.410902738571167, 0.6838463544845581, 0.4198354482650757, -0.6969732642173767, -0.14180786907672882, 0.48462235927581787, -0.7340816855430603, -0.09603086113929749, 0.5138707756996155, 0.9006104469299316, 1.052554726600647, -0.04387497156858444, -0.6620950698852539, 0.29913318157196045, 0.5081597566604614, 0.12850642204284668, 0.4417377710342407, 0.7943270802497864, -0.7073273062705994, -0.07511018216609955, 0.0191994309425354, 0.5199860334396362, 0.8791877031326294, -0.8322564363479614, -0.7350091934204102, -1.0920594930648804, 0.8753132224082947, 0.35002511739730835, -0.5546954870223999, -0.4522245526313782, -0.1298009753227234, -0.957366406917572, 0.20797111093997955, -0.5738971829414368, 0.06666096299886703, 0.3955097198486328, -0.26859503984451294, -0.2987530529499054],
|
|
"long_output_samples": [-1.125, 0.3125, 0.75, -0.9375, 0.5, 0.5625, -0.3125, 0.125, 0.75, -0.125, -0.06733516603708267, -0.9318115711212158, -0.4943115711212158, 0.12016483396291733, -0.7443115711212158, -0.681811511516571, 0.7451648116111755, -1.1193115711212158, -0.4943115711212158, 0.9326648116111755, -0.5526970028877258, 0.7137718796730042, -0.9278546571731567, -0.3859345316886902, 0.47903838753700256, 0.5415382981300354, 0.2160443514585495, 0.6197132468223572, -0.9278546571731567, 0.4035443365573883, 0.6819117665290833, 0.09392789751291275, 0.4912314713001251, 0.3996358811855316, 0.26871275901794434, 0.33121272921562195, -0.4914110600948334, -0.0792417898774147, 0.4912314713001251, -0.31433215737342834]
|
|
},
|
|
{
|
|
"name": "mscale_without_all_dim",
|
|
"max_position_embeddings": 131072,
|
|
"rope_scaling": {
|
|
"rope_type": "yarn",
|
|
"factor": 4.0,
|
|
"original_max_position_embeddings": 32768,
|
|
"mscale": 2.0
|
|
},
|
|
"inv_freq": [1.0, 0.8146172165870667, 0.663601279258728, 0.5405809879302979, 0.44036662578582764, 0.3587302267551422, 0.2922278344631195, 0.2380538135766983, 0.193922758102417, 0.1579728126525879, 0.12868738174438477, 0.10483095049858093, 0.08539710193872452, 0.06956595182418823, 0.05666961893439293, 0.046164050698280334, 0.03760603070259094, 0.030634520575404167, 0.02495540864765644, 0.020329104736447334, 0.016560440883040428, 0.013490419834852219, 0.010989529080688953, 0.008952259086072445, 0.00729266507551074, 0.005693200044333935, 0.004436136223375797, 0.0034494914580136538, 0.0026762052439153194, 0.0020710784010589123, 0.0015983396442607045, 0.0012296997010707855, 0.0009428090415894985, 0.0007200266700237989, 0.000547443050891161, 0.00041410254198126495, 0.0003113861894235015, 0.00023252220125868917, 0.0001721968874335289, 0.00012624710507225245, 9.141606278717518e-05, 6.516046414617449e-05, 4.549785808194429e-05, 3.706333882291801e-05, 3.0192435588105582e-05, 2.4595277864136733e-05, 2.003573717956897e-05, 1.6321455404977314e-05, 1.3295739336172119e-05, 1.0830938663275447e-05, 8.823069038044196e-06, 7.187423761934042e-06, 5.854999471921474e-06, 4.7695834837213624e-06, 3.8853850128361955e-06, 3.1651015888201073e-06, 2.578346311565838e-06, 2.100365236401558e-06, 1.7109938426074223e-06, 1.3938050642536837e-06, 1.1354176194799948e-06, 9.249307595382561e-07, 7.5346451922087e-07, 6.137851755738666e-07],
|
|
"attention_factor": 1.138629436111989,
|
|
"output_samples": [-1.125, 0.3125, 0.75, -0.9375, 0.5, 0.5625, -0.3125, 0.125, 0.75, -0.125, -0.06660930067300797, -0.9326654672622681, -0.49516549706459045, 0.12089069932699203, -0.7451654672622681, -0.6826654672622681, 0.7458906769752502, -1.120165467262268, -0.49516549706459045, 0.9333906769752502, -0.5575152039527893, 0.7235569357872009, -0.9375719428062439, -0.38849642872810364, 0.49885261058807373, 0.5613526105880737, 0.21633166074752808, 0.623228132724762, -0.9375719428062439, 0.4038316607475281, 0.6961049437522888, 0.09358730167150497, 0.49438023567199707, 0.4224497079849243, 0.26966211199760437, 0.33216211199760437, -0.4962541162967682, -0.08124681562185287, 0.49438023567199707, -0.31806203722953796],
|
|
"key_samples": [-0.9542381763458252, 0.2650661766529083, 0.63615882396698, -0.7951985001564026, 0.4241058826446533, 0.4771190881729126, -0.2650661766529083, 0.10602647066116333, 0.63615882396698, -0.10602647066116333, 0.4762541353702545, -0.8254690766334534, -0.43556439876556396, 0.10932422429323196, -0.6557060480117798, -0.3707357347011566, 0.6483418345451355, -0.9842557907104492, -0.4371270537376404, 0.8196315169334412, -1.0515915155410767, 0.7187532186508179, -0.9618231654167175, -0.392868310213089, 0.899381160736084, 0.35112303495407104, 0.21406617760658264, 0.5513875484466553, -0.9558455348014832, 0.3372723460197449, -0.7504856586456299, 0.11251698434352875, 0.41225114464759827, 0.9217395782470703, 0.2049589604139328, 0.30517345666885376, -0.4501397907733917, -0.09262187033891678, 0.41117656230926514, -0.3074323236942291],
|
|
"long_key_samples": [-0.7985695600509644, -0.3461228013038635, -0.41708531975746155, 0.6941357851028442, 0.42615240812301636, -0.7074602246284485, -0.14394155144691467, 0.49191415309906006, -0.7451270222663879, -0.09747575968503952, 0.5216026306152344, 0.9141613245010376, 1.0683919191360474, -0.04453512281179428, -0.6720572113990784, 0.30363404750823975, 0.5158056616783142, 0.13043993711471558, 0.44838428497314453, 0.8062787652015686, -0.7179700136184692, -0.07624031603336334, 0.01948830485343933, 0.5278098583221436, 0.89241623878479, -0.8447787761688232, -0.7460684180259705, -1.1084909439086914, 0.888483464717865, 0.3552916944026947, -0.563041627407074, -0.4590288996696472, -0.13175401091575623, -0.9717712998390198, 0.21110032498836517, -0.5825321674346924, 0.06766396760940552, 0.4014606773853302, -0.27263641357421875, -0.30324819684028625],
|
|
"long_output_samples": [-1.125, 0.3125, 0.75, -0.9375, 0.5, 0.5625, -0.3125, 0.125, 0.75, -0.125, -0.06660927832126617, -0.9326655864715576, -0.4951655864715576, 0.12089072912931442, -0.7451655864715576, -0.6826655864715576, 0.745890736579895, -1.1201655864715576, -0.4951655864715576, 0.9333907961845398, -0.5575114488601685, 0.7235738039016724, -0.9375671744346619, -0.38849249482154846, 0.49878236651420593, 0.5612823367118835, 0.21633726358413696, 0.6232365965843201, -0.9375671744346619, 0.40383729338645935, 0.696094810962677, 0.09358564019203186, 0.4943827986717224, 0.4224432110786438, 0.2696602940559387, 0.33216026425361633, -0.4962575137615204, -0.08124911040067673, 0.4943827986717224, -0.31806522607803345]
|
|
},
|
|
{
|
|
"name": "attention_override",
|
|
"max_position_embeddings": 131072,
|
|
"rope_scaling": {
|
|
"rope_type": "yarn",
|
|
"factor": 4.0,
|
|
"original_max_position_embeddings": 32768,
|
|
"mscale": 2.0,
|
|
"mscale_all_dim": 1.0,
|
|
"attention_factor": 1.2
|
|
},
|
|
"inv_freq": [1.0, 0.8146172165870667, 0.663601279258728, 0.5405809879302979, 0.44036662578582764, 0.3587302267551422, 0.2922278344631195, 0.2380538135766983, 0.193922758102417, 0.1579728126525879, 0.12868738174438477, 0.10483095049858093, 0.08539710193872452, 0.06956595182418823, 0.05666961893439293, 0.046164050698280334, 0.03760603070259094, 0.030634520575404167, 0.02495540864765644, 0.020329104736447334, 0.016560440883040428, 0.013490419834852219, 0.010989529080688953, 0.008952259086072445, 0.00729266507551074, 0.005693200044333935, 0.004436136223375797, 0.0034494914580136538, 0.0026762052439153194, 0.0020710784010589123, 0.0015983396442607045, 0.0012296997010707855, 0.0009428090415894985, 0.0007200266700237989, 0.000547443050891161, 0.00041410254198126495, 0.0003113861894235015, 0.00023252220125868917, 0.0001721968874335289, 0.00012624710507225245, 9.141606278717518e-05, 6.516046414617449e-05, 4.549785808194429e-05, 3.706333882291801e-05, 3.0192435588105582e-05, 2.4595277864136733e-05, 2.003573717956897e-05, 1.6321455404977314e-05, 1.3295739336172119e-05, 1.0830938663275447e-05, 8.823069038044196e-06, 7.187423761934042e-06, 5.854999471921474e-06, 4.7695834837213624e-06, 3.8853850128361955e-06, 3.1651015888201073e-06, 2.578346311565838e-06, 2.100365236401558e-06, 1.7109938426074223e-06, 1.3938050642536837e-06, 1.1354176194799948e-06, 9.249307595382561e-07, 7.5346451922087e-07, 6.137851755738666e-07],
|
|
"attention_factor": 1.2,
|
|
"output_samples": [-1.125, 0.3125, 0.75, -0.9375, 0.5, 0.5625, -0.3125, 0.125, 0.75, -0.125, -0.06472662836313248, -0.9348804354667664, -0.49738040566444397, 0.12277336418628693, -0.7473803758621216, -0.6848803758621216, 0.7477733492851257, -1.1223803758621216, -0.49738040566444397, 0.9352733492851257, -0.5723831057548523, 0.7537875175476074, -0.9680386185646057, -0.3967573642730713, 0.5678153038024902, 0.6303153038024902, 0.21607981622219086, 0.632787823677063, -0.9680386185646057, 0.40357980132102966, 0.7441161274909973, 0.0914464220404625, 0.503157913684845, 0.5066576600074768, 0.27139753103256226, 0.33389753103256226, -0.5114732980728149, -0.08818687498569489, 0.503157913684845, -0.32998526096343994],
|
|
"key_samples": [-1.0056703090667725, 0.2793528735637665, 0.6704469323158264, -0.8380586504936218, 0.4469646215438843, 0.5028351545333862, -0.2793528735637665, 0.11174115538597107, 0.6704469323158264, -0.11174115538597107, 0.5019235610961914, -0.8699607849121094, -0.45904073119163513, 0.11521665006875992, -0.691047728061676, -0.39071786403656006, 0.6832866072654724, -1.0373058319091797, -0.4606876075267792, 0.8638086318969727, -1.1082708835601807, 0.7574930787086487, -1.0136642456054688, -0.4140433967113495, 0.9478565454483032, 0.3700481057167053, 0.2256040722131729, 0.5811066627502441, -1.0073643922805786, 0.3554508686065674, -0.7909358739852905, 0.11858150362968445, 0.43447092175483704, 0.9714200496673584, 0.21600596606731415, 0.32162195444107056, -0.4744017422199249, -0.09761406481266022, 0.4333384037017822, -0.3240025043487549],
|
|
"long_key_samples": [-0.8416112661361694, -0.3647783398628235, -0.43956565856933594, 0.7315487861633301, 0.4491214454174042, -0.7455912828445435, -0.15169979631900787, 0.5184276700019836, -0.7852883338928223, -0.10272957384586334, 0.5497163534164429, 0.9634333848953247, 1.125976800918579, -0.04693550616502762, -0.7082802057266235, 0.3199995160102844, 0.5436069369316101, 0.137470543384552, 0.4725515842437744, 0.8497360944747925, -0.7566676139831543, -0.08034956455230713, 0.020538747310638428, 0.5562581419944763, 0.9405162930488586, -0.8903112411499023, -0.7862804532051086, -1.1682372093200684, 0.9363715648651123, 0.3744414448738098, -0.5933887958526611, -0.483769953250885, -0.13885538280010223, -1.0241484642028809, 0.22247834503650665, -0.6139299273490906, 0.07131095230579376, 0.4230988621711731, -0.28733116388320923, -0.31959283351898193],
|
|
"long_output_samples": [-1.125, 0.3125, 0.75, -0.9375, 0.5, 0.5625, -0.3125, 0.125, 0.75, -0.125, -0.06472662091255188, -0.9348804950714111, -0.49738049507141113, 0.12277338653802872, -0.7473804950714111, -0.6848804354667664, 0.7477734684944153, -1.1223804950714111, -0.49738049507141113, 0.9352734684944153, -0.572378396987915, 0.7538023591041565, -0.9680330753326416, -0.39675262570381165, 0.5677414536476135, 0.6302414536476135, 0.21608571708202362, 0.6327958703041077, -0.9680330753326416, 0.4035857617855072, 0.7441056370735168, 0.09144478291273117, 0.5031596422195435, 0.5066503882408142, 0.2713957726955414, 0.3338957726955414, -0.5114761590957642, -0.08818896859884262, 0.5031596422195435, -0.3299880027770996]
|
|
},
|
|
{
|
|
"name": "zero_attention",
|
|
"max_position_embeddings": 131072,
|
|
"rope_scaling": {
|
|
"rope_type": "yarn",
|
|
"factor": 4.0,
|
|
"original_max_position_embeddings": 32768,
|
|
"attention_factor": 0.0
|
|
},
|
|
"inv_freq": [1.0, 0.8146172165870667, 0.663601279258728, 0.5405809879302979, 0.44036662578582764, 0.3587302267551422, 0.2922278344631195, 0.2380538135766983, 0.193922758102417, 0.1579728126525879, 0.12868738174438477, 0.10483095049858093, 0.08539710193872452, 0.06956595182418823, 0.05666961893439293, 0.046164050698280334, 0.03760603070259094, 0.030634520575404167, 0.02495540864765644, 0.020329104736447334, 0.016560440883040428, 0.013490419834852219, 0.010989529080688953, 0.008952259086072445, 0.00729266507551074, 0.005693200044333935, 0.004436136223375797, 0.0034494914580136538, 0.0026762052439153194, 0.0020710784010589123, 0.0015983396442607045, 0.0012296997010707855, 0.0009428090415894985, 0.0007200266700237989, 0.000547443050891161, 0.00041410254198126495, 0.0003113861894235015, 0.00023252220125868917, 0.0001721968874335289, 0.00012624710507225245, 9.141606278717518e-05, 6.516046414617449e-05, 4.549785808194429e-05, 3.706333882291801e-05, 3.0192435588105582e-05, 2.4595277864136733e-05, 2.003573717956897e-05, 1.6321455404977314e-05, 1.3295739336172119e-05, 1.0830938663275447e-05, 8.823069038044196e-06, 7.187423761934042e-06, 5.854999471921474e-06, 4.7695834837213624e-06, 3.8853850128361955e-06, 3.1651015888201073e-06, 2.578346311565838e-06, 2.100365236401558e-06, 1.7109938426074223e-06, 1.3938050642536837e-06, 1.1354176194799948e-06, 9.249307595382561e-07, 7.5346451922087e-07, 6.137851755738666e-07],
|
|
"attention_factor": 0.0,
|
|
"output_samples": [-1.125, 0.3125, 0.75, -0.9375, 0.5, 0.5625, -0.3125, 0.125, 0.75, -0.125, -0.59375, -0.3125, 0.125, -0.40625, -0.125, -0.0625, 0.21875, -0.5, 0.125, 0.40625, -0.0078125, -0.015625, -0.15625, -0.109375, -0.1171875, -0.0546875, -0.0625, 0.0859375, -0.15625, 0.125, 0.05078125, -0.1015625, 0.046875, -0.05078125, -0.05859375, 0.00390625, -0.00390625, 0.0, 0.046875, 0.0390625],
|
|
"key_samples": [-0.0, 0.0, 0.0, -0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -0.0, 0.0, 0.0, -0.0, -0.0, 0.0, -0.0, 0.0, 0.0, -0.0, 0.0, -0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -0.0, 0.0, -0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
|
|
"long_key_samples": [-0.0, -0.0, -0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -0.0, -0.0, -0.0, 0.0, 0.0, 0.0, -0.0, 0.0, -0.0, 0.0, -0.0, 0.0, 0.0, 0.0, 0.0],
|
|
"long_output_samples": [-1.125, 0.3125, 0.75, -0.9375, 0.5, 0.5625, -0.3125, 0.125, 0.75, -0.125, -0.59375, -0.3125, 0.125, -0.40625, -0.125, -0.0625, 0.21875, -0.5, 0.125, 0.40625, -0.0078125, -0.015625, -0.15625, -0.109375, -0.1171875, -0.0546875, -0.0625, 0.0859375, -0.15625, 0.125, 0.05078125, -0.1015625, 0.046875, -0.05078125, -0.05859375, 0.00390625, -0.00390625, 0.0, 0.046875, 0.0390625]
|
|
},
|
|
{
|
|
"name": "factor_one",
|
|
"max_position_embeddings": 32768,
|
|
"rope_scaling": {
|
|
"rope_type": "yarn",
|
|
"factor": 1.0,
|
|
"original_max_position_embeddings": 32768
|
|
},
|
|
"inv_freq": [1.0, 0.8146172165870667, 0.663601279258728, 0.5405809879302979, 0.44036662578582764, 0.3587302267551422, 0.2922278344631195, 0.2380538135766983, 0.193922758102417, 0.1579728126525879, 0.12868738174438477, 0.10483095049858093, 0.08539710193872452, 0.06956595182418823, 0.05666961893439293, 0.046164050698280334, 0.03760603070259094, 0.030634520575404167, 0.02495540864765644, 0.020329104736447334, 0.016560440883040428, 0.013490419834852219, 0.010989529080688953, 0.008952259086072445, 0.00729266507551074, 0.005940730683505535, 0.00483942124992609, 0.003942275885492563, 0.0032114461064338684, 0.0026160990819334984, 0.002131119603291154, 0.0017360467463731766, 0.001414213445968926, 0.001152042648755014, 0.0009384738514199853, 0.0007644969737157226, 0.0006227724370546639, 0.0005073211505077779, 0.0004132725589442998, 0.00033665893715806305, 0.00027424818836152554, 0.00022340727446135134, 0.00018199143232777715, 0.00014825335529167205, 0.00012076974235242233, 9.838111145654693e-05, 8.014294871827587e-05, 6.528582161990926e-05, 5.3182957344688475e-05, 4.332375465310179e-05, 3.529227615217678e-05, 2.8749695047736168e-05, 2.3419997887685895e-05, 1.907833393488545e-05, 1.5541540051344782e-05, 1.2660406355280429e-05, 1.0313385246263351e-05, 8.401460945606232e-06, 6.843975370429689e-06, 5.575220257014735e-06, 4.541670477919979e-06, 3.6997230381530244e-06, 3.01385807688348e-06, 2.4551407022954663e-06],
|
|
"attention_factor": 1.0,
|
|
"output_samples": [-1.125, 0.3125, 0.75, -0.9375, 0.5, 0.5625, -0.3125, 0.125, 0.75, -0.125, -0.07698158919811249, -0.9204629063606262, -0.48296287655830383, 0.11051841825246811, -0.7329629063606262, -0.6704628467559814, 0.7355184555053711, -1.107962965965271, -0.48296287655830383, 0.9230184555053711, -0.5068065524101257, 0.6231921911239624, -0.8396487832069397, -0.3629710078239441, 0.33127108216285706, 0.39377105236053467, 0.20752035081386566, 0.5804018378257751, -0.8396487832069397, 0.3950203061103821, 0.5698629021644592, 0.09191276878118515, 0.4576781094074249, 0.24600763618946075, 0.2542138695716858, 0.3167138695716858, -0.446920245885849, -0.06380360573530197, 0.4576781094074249, -0.2809051275253296],
|
|
"key_samples": [-0.838058590888977, 0.23279406130313873, 0.558705747127533, -0.6983821988105774, 0.372470498085022, 0.4190292954444885, -0.23279406130313873, 0.0931176245212555, 0.558705747127533, -0.0931176245212555, 0.41826963424682617, -0.7249672412872314, -0.3820733428001404, 0.09608406573534012, -0.5758744478225708, -0.32559821009635925, 0.5694054961204529, -0.8646252155303955, -0.38388878107070923, 0.7198393940925598, -0.9235590696334839, 0.6312442421913147, -0.8465202450752258, -0.3439614772796631, 0.7898766398429871, 0.3083733916282654, 0.18800339102745056, 0.4811018109321594, -0.839911162853241, 0.29621919989585876, -0.6591131687164307, 0.09881791472434998, 0.3626976013183594, 0.8085233569145203, 0.18001241981983185, 0.2680182456970215, -0.39533478021621704, -0.07844901829957962, 0.363334059715271, -0.26999711990356445],
|
|
"long_key_samples": [-0.7013427019119263, -0.3039819300174713, 0.35973766446113586, 0.39532166719436646, 0.3787490427494049, -0.6213260889053345, -0.12641650438308716, 0.43750637769699097, -0.8022534847259521, -0.06288349628448486, 0.4580969214439392, 0.8028610944747925, 0.2806074321269989, 0.07588721811771393, -0.6318604946136475, 0.2666662335395813, 0.45300573110580444, -0.9026716947555542, 0.38838621973991394, 0.6712319850921631, -0.6305563449859619, -0.0669579803943634, -0.9541407823562622, 0.6678540110588074, 0.7635167241096497, -0.7419260144233704, -0.6552337408065796, -0.1940896213054657, 0.6145985126495361, 0.3587382435798645, -0.4944906234741211, -0.403141587972641, 0.32485759258270264, -0.8862977027893066, 0.2011280655860901, -0.5116082429885864, 0.059425801038742065, 0.17936371266841888, 0.014071017503738403, -0.25465747714042664],
|
|
"long_output_samples": [-1.125, 0.3125, 0.75, -0.9375, 0.5, 0.5625, -0.3125, 0.125, 0.75, -0.125, -0.0769815519452095, -0.9204629063606262, -0.4829629063606262, 0.1105184480547905, -0.7329629063606262, -0.6704629063606262, 0.7355184555053711, -1.107962965965271, -0.4829629063606262, 0.9230184555053711, -0.5068061351776123, 0.6232134103775024, -0.8396475911140442, -0.36297014355659485, 0.33121639490127563, 0.39371639490127563, 0.20752526819705963, 0.5804116725921631, -0.8396475911140442, 0.39502525329589844, 0.5698559880256653, 0.09191132336854935, 0.45768284797668457, 0.24600550532341003, 0.25421223044395447, 0.31671223044395447, -0.44692519307136536, -0.06380632519721985, 0.45768284797668457, -0.2809096872806549]
|
|
},
|
|
{
|
|
"name": "factor_one_attention",
|
|
"max_position_embeddings": 32768,
|
|
"rope_scaling": {
|
|
"rope_type": "yarn",
|
|
"factor": 1.0,
|
|
"original_max_position_embeddings": 32768,
|
|
"attention_factor": 1.2
|
|
},
|
|
"inv_freq": [1.0, 0.8146172165870667, 0.663601279258728, 0.5405809879302979, 0.44036662578582764, 0.3587302267551422, 0.2922278344631195, 0.2380538135766983, 0.193922758102417, 0.1579728126525879, 0.12868738174438477, 0.10483095049858093, 0.08539710193872452, 0.06956595182418823, 0.05666961893439293, 0.046164050698280334, 0.03760603070259094, 0.030634520575404167, 0.02495540864765644, 0.020329104736447334, 0.016560440883040428, 0.013490419834852219, 0.010989529080688953, 0.008952259086072445, 0.00729266507551074, 0.005940730683505535, 0.00483942124992609, 0.003942275885492563, 0.0032114461064338684, 0.0026160990819334984, 0.002131119603291154, 0.0017360467463731766, 0.001414213445968926, 0.001152042648755014, 0.0009384738514199853, 0.0007644969737157226, 0.0006227724370546639, 0.0005073211505077779, 0.0004132725589442998, 0.00033665893715806305, 0.00027424818836152554, 0.00022340727446135134, 0.00018199143232777715, 0.00014825335529167205, 0.00012076974235242233, 9.838111145654693e-05, 8.014294871827587e-05, 6.528582161990926e-05, 5.3182957344688475e-05, 4.332375465310179e-05, 3.529227615217678e-05, 2.8749695047736168e-05, 2.3419997887685895e-05, 1.907833393488545e-05, 1.5541540051344782e-05, 1.2660406355280429e-05, 1.0313385246263351e-05, 8.401460945606232e-06, 6.843975370429689e-06, 5.575220257014735e-06, 4.541670477919979e-06, 3.6997230381530244e-06, 3.01385807688348e-06, 2.4551407022954663e-06],
|
|
"attention_factor": 1.2,
|
|
"output_samples": [-1.125, 0.3125, 0.75, -0.9375, 0.5, 0.5625, -0.3125, 0.125, 0.75, -0.125, -0.06472670286893845, -0.9348803758621216, -0.4973803758621216, 0.12277330458164215, -0.7473803758621216, -0.6848803758621216, 0.747773289680481, -1.1223803758621216, -0.4973803758621216, 0.9352733492851257, -0.5723469257354736, 0.7537283897399902, -0.9681026935577393, -0.3967021107673645, 0.5676707029342651, 0.6301706433296204, 0.2161395400762558, 0.6327927112579346, -0.9681026935577393, 0.4036395251750946, 0.7438934445381165, 0.0915277972817421, 0.5031976103782654, 0.5063737034797668, 0.2714727520942688, 0.3339727520942688, -0.5114141702651978, -0.0881088525056839, 0.5031976103782654, -0.3299192190170288],
|
|
"key_samples": [-1.0056703090667725, 0.2793528735637665, 0.6704469323158264, -0.8380586504936218, 0.4469646215438843, 0.5028351545333862, -0.2793528735637665, 0.11174115538597107, 0.6704469323158264, -0.11174115538597107, 0.5019235610961914, -0.8699607849121094, -0.45848801732063293, 0.11530087888240814, -0.691049337387085, -0.39071786403656006, 0.6832866072654724, -1.0375502109527588, -0.4606665372848511, 0.8638073801994324, -1.1082708835601807, 0.7574930787086487, -1.0158244371414185, -0.41275376081466675, 0.9478520154953003, 0.3700481057167053, 0.2256040722131729, 0.577322244644165, -1.007893443107605, 0.35546308755874634, -0.7909358739852905, 0.11858150362968445, 0.4352371394634247, 0.9702280163764954, 0.21601492166519165, 0.32162195444107056, -0.4744017422199249, -0.09413881599903107, 0.43600091338157654, -0.32399654388427734],
|
|
"long_key_samples": [-0.8416112661361694, -0.3647783398628235, 0.431685209274292, 0.4743860065937042, 0.45449885725975037, -0.7455912828445435, -0.15169979631900787, 0.5250077247619629, -0.9627041816711426, -0.07546019554138184, 0.5497163534164429, 0.9634333848953247, 0.33672896027565, 0.09106464684009552, -0.7582326531410217, 0.3199995160102844, 0.5436069369316101, -1.083206057548523, 0.4660634994506836, 0.8054785132408142, -0.7566676139831543, -0.08034956455230713, -1.1449689865112305, 0.8014248609542847, 0.9162201285362244, -0.8903112411499023, -0.7862804532051086, -0.23290759325027466, 0.7375181913375854, 0.43048590421676636, -0.5933887958526611, -0.483769953250885, 0.3898290991783142, -1.0635573863983154, 0.24135369062423706, -0.6139299273490906, 0.07131095230579376, 0.21523645520210266, 0.016885221004486084, -0.3055889308452606],
|
|
"long_output_samples": [-1.125, 0.3125, 0.75, -0.9375, 0.5, 0.5625, -0.3125, 0.125, 0.75, -0.125, -0.06472668051719666, -0.9348803758621216, -0.4973803758621216, 0.12277331948280334, -0.7473803758621216, -0.6848803758621216, 0.747773289680481, -1.1223803758621216, -0.4973803758621216, 0.9352733492851257, -0.572342038154602, 0.7537431716918945, -0.9680968523025513, -0.39669719338417053, 0.5675956010818481, 0.6300956010818481, 0.21614553034305573, 0.6328007578849792, -0.9680968523025513, 0.40364551544189453, 0.7438828945159912, 0.09152614325284958, 0.5031993389129639, 0.5063661336898804, 0.27147096395492554, 0.33397096395492554, -0.5114169716835022, -0.08811095356941223, 0.5031993389129639, -0.3299219310283661]
|
|
}
|
|
]
|
|
}
|