diff --git a/lm-evaluation-harness b/lm-evaluation-harness index 67a990e..543617f 160000 --- a/lm-evaluation-harness +++ b/lm-evaluation-harness @@ -1 +1 @@ -Subproject commit 67a990e7345d4ba940e8281ac7c9113ccef2a446 +Subproject commit 543617fef9ba885e87f8db8930fbbff1d4e2ca49 diff --git a/metadata/model_scores/results_01-ai__Yi-34B.json b/metadata/model_scores/results_01-ai__Yi-34B.json index 1e7808e..f488d2f 100644 --- a/metadata/model_scores/results_01-ai__Yi-34B.json +++ b/metadata/model_scores/results_01-ai__Yi-34B.json @@ -1,6 +1,6 @@ { "model_name": "01-ai/Yi-34B", - "last_updated": "2024-12-04 11:23:57.995135", + "last_updated": "2024-12-19 13:39:23.602577", "results": { "harness": { "arc:challenge": { @@ -561,289 +561,403 @@ "timestamp": "2024-11-27T03-38-49.243482" }, "mmlu_world_religions": { - "acc": 0.8654970760233918, - "acc_stderr": 0.026168221344662294, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.8713450292397661, + "acc_stderr": 0.02567934272327694, + "brier_score": 0.18327084828640616, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_formal_logic": { - "acc": 0.5555555555555556, - "acc_stderr": 0.04444444444444449, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.5317460317460317, + "acc_stderr": 0.04463112720677172, + "brier_score": 0.586843042091581, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_prehistory": { - "acc": 0.8734567901234568, - "acc_stderr": 0.018498600558790917, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.8487654320987654, + "acc_stderr": 0.019935086092149876, + "brier_score": 0.22676479805196784, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_moral_scenarios": { - "acc": 0.6458100558659218, - "acc_stderr": 0.015995644947299225, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.43687150837988825, + "acc_stderr": 0.016588680864530626, + "brier_score": 0.6715835727205642, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_high_school_world_history": { - "acc": 0.9240506329113924, - "acc_stderr": 0.017244633251065688, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.890295358649789, + "acc_stderr": 0.020343400734868847, + "brier_score": 0.16060559995313187, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_moral_disputes": { - "acc": 0.8323699421965318, - "acc_stderr": 0.02011057991973484, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.8005780346820809, + "acc_stderr": 0.021511900654252555, + "brier_score": 0.2933084466158301, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_professional_law": { - "acc": 0.5971316818774446, - "acc_stderr": 0.01252695557711801, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.6016949152542372, + "acc_stderr": 0.012503310565166239, + "brier_score": 0.5424167959759115, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_logical_fallacies": { - "acc": 0.8834355828220859, - "acc_stderr": 0.025212327210507094, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.8711656441717791, + "acc_stderr": 0.026321383198783636, + "brier_score": 0.20225943773288335, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_high_school_us_history": { - "acc": 0.9117647058823529, - "acc_stderr": 0.019907399791316942, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.9019607843137255, + "acc_stderr": 0.020871118455552097, + "brier_score": 0.14855776488105746, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_philosophy": { - "acc": 0.8392282958199357, - "acc_stderr": 0.02086238808239191, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.8070739549839229, + "acc_stderr": 0.02241151678091136, + "brier_score": 0.26327499807735083, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_jurisprudence": { - "acc": 0.8981481481481481, - "acc_stderr": 0.02923927267563274, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.9259259259259259, + "acc_stderr": 0.025317997297209738, + "brier_score": 0.1481508666174445, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_international_law": { - "acc": 0.9173553719008265, - "acc_stderr": 0.02513538235660422, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.859504132231405, + "acc_stderr": 0.031722334260021585, + "brier_score": 0.17724953898088328, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_high_school_european_history": { - "acc": 0.8666666666666667, - "acc_stderr": 0.026544435312706473, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.8545454545454545, + "acc_stderr": 0.02753019635506658, + "brier_score": 0.2116949199581668, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_high_school_government_and_politics": { - "acc": 0.9792746113989638, - "acc_stderr": 0.010281417011909025, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.9689119170984456, + "acc_stderr": 0.012525310625527019, + "brier_score": 0.046998485404674356, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_high_school_microeconomics": { - "acc": 0.8529411764705882, - "acc_stderr": 0.023005459446673957, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.8235294117647058, + "acc_stderr": 0.024762902678057926, + "brier_score": 0.22790712051886977, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_high_school_geography": { - "acc": 0.8888888888888888, - "acc_stderr": 0.022390787638216763, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.9141414141414141, + "acc_stderr": 0.019960225563172885, + "brier_score": 0.1433233477967982, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_high_school_psychology": { - "acc": 0.9155963302752294, - "acc_stderr": 0.011918819327334889, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.9027522935779817, + "acc_stderr": 0.01270353340854038, + "brier_score": 0.14856247232733205, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_public_relations": { - "acc": 0.7363636363636363, - "acc_stderr": 0.04220224692971987, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.7545454545454545, + "acc_stderr": 0.04122066502878284, + "brier_score": 0.3290300957956976, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_us_foreign_policy": { - "acc": 0.91, - "acc_stderr": 0.028762349126466125, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.94, + "acc_stderr": 0.0238683256575942, + "brier_score": 0.10569846644231891, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_sociology": { - "acc": 0.8855721393034826, - "acc_stderr": 0.022509345325101703, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.8756218905472637, + "acc_stderr": 0.023335401790166327, + "brier_score": 0.1946006826671885, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_high_school_macroeconomics": { - "acc": 0.7871794871794872, - "acc_stderr": 0.020752423722128013, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.7974358974358975, + "acc_stderr": 0.020377660970371393, + "brier_score": 0.28320335308971284, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_security_studies": { - "acc": 0.8530612244897959, - "acc_stderr": 0.02266540041721763, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.8081632653061225, + "acc_stderr": 0.02520696315422539, + "brier_score": 0.2731915635613184, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_professional_psychology": { - "acc": 0.8235294117647058, - "acc_stderr": 0.015422512066262547, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.7990196078431373, + "acc_stderr": 0.01621193888965559, + "brier_score": 0.27985172892589205, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_human_sexuality": { - "acc": 0.8625954198473282, - "acc_stderr": 0.030194823996804475, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.8244274809160306, + "acc_stderr": 0.03336820338476076, + "brier_score": 0.21410963942217057, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_econometrics": { - "acc": 0.5526315789473685, - "acc_stderr": 0.04677473004491199, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.6140350877192983, + "acc_stderr": 0.04579639422070434, + "brier_score": 0.5066114633945582, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_miscellaneous": { - "acc": 0.9054916985951469, - "acc_stderr": 0.010461015338193071, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.896551724137931, + "acc_stderr": 0.010890452544691481, + "brier_score": 0.14592820623528896, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_marketing": { - "acc": 0.9273504273504274, - "acc_stderr": 0.017004368568132353, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.9145299145299145, + "acc_stderr": 0.018315891685625838, + "brier_score": 0.1259947955560078, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_management": { - "acc": 0.912621359223301, - "acc_stderr": 0.027960689125970658, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.8640776699029126, + "acc_stderr": 0.033932957297610096, + "brier_score": 0.16745701098951601, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_nutrition": { - "acc": 0.869281045751634, - "acc_stderr": 0.019301873624215298, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.8398692810457516, + "acc_stderr": 0.0209987409303623, + "brier_score": 0.22269453920155644, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_medical_genetics": { - "acc": 0.87, - "acc_stderr": 0.03379976689896309, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.89, + "acc_stderr": 0.03144660377352202, + "brier_score": 0.18956379020345207, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_human_aging": { - "acc": 0.7892376681614349, - "acc_stderr": 0.027373095500540186, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.7668161434977578, + "acc_stderr": 0.028380391147094713, + "brier_score": 0.3350636038155653, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_professional_medicine": { - "acc": 0.8235294117647058, - "acc_stderr": 0.02315746830855936, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.8088235294117647, + "acc_stderr": 0.023886881922440352, + "brier_score": 0.2711944775698018, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_college_medicine": { - "acc": 0.7052023121387283, - "acc_stderr": 0.03476599607516478, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.6878612716763006, + "acc_stderr": 0.03533133389323657, + "brier_score": 0.37305027047556816, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_business_ethics": { - "acc": 0.79, - "acc_stderr": 0.040936018074033256, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.77, + "acc_stderr": 0.04229525846816507, + "brier_score": 0.2994103903421135, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_clinical_knowledge": { - "acc": 0.7924528301886793, - "acc_stderr": 0.02495991802891127, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.7811320754716982, + "acc_stderr": 0.0254478638251086, + "brier_score": 0.27156114148483634, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_global_facts": { - "acc": 0.52, - "acc_stderr": 0.050211673156867795, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.62, + "acc_stderr": 0.04878317312145634, + "brier_score": 0.5321935454798382, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_virology": { "acc": 0.572289156626506, "acc_stderr": 0.038515976837185335, - "timestamp": "2024-11-27T03-38-49.243482" + "brier_score": 0.6847042867927003, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_professional_accounting": { - "acc": 0.6631205673758865, - "acc_stderr": 0.028195534873966734, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.6099290780141844, + "acc_stderr": 0.029097675599463926, + "brier_score": 0.47774400145574714, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_college_physics": { - "acc": 0.4803921568627451, - "acc_stderr": 0.04971358884367406, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.5, + "acc_stderr": 0.04975185951049946, + "brier_score": 0.6111197699586813, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_high_school_physics": { - "acc": 0.5231788079470199, - "acc_stderr": 0.04078093859163084, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.5099337748344371, + "acc_stderr": 0.04081677107248436, + "brier_score": 0.6474363239128381, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_high_school_biology": { - "acc": 0.8806451612903226, - "acc_stderr": 0.01844341132531541, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.8774193548387097, + "acc_stderr": 0.018656720991789402, + "brier_score": 0.18133846283863253, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_college_biology": { - "acc": 0.875, - "acc_stderr": 0.02765610492929436, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.8611111111111112, + "acc_stderr": 0.028919802956134905, + "brier_score": 0.19358888814122588, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_anatomy": { - "acc": 0.7481481481481481, - "acc_stderr": 0.037498507091740206, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.7555555555555555, + "acc_stderr": 0.037125378336148665, + "brier_score": 0.34125616897932837, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_college_chemistry": { - "acc": 0.48, + "acc": 0.52, "acc_stderr": 0.050211673156867795, - "timestamp": "2024-11-27T03-38-49.243482" + "brier_score": 0.5641430565145782, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_computer_security": { - "acc": 0.82, - "acc_stderr": 0.03861229196653695, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.81, + "acc_stderr": 0.039427724440366234, + "brier_score": 0.23630050012470782, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_college_computer_science": { "acc": 0.64, "acc_stderr": 0.04824181513244218, - "timestamp": "2024-11-27T03-38-49.243482" + "brier_score": 0.4467921110077767, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_astronomy": { - "acc": 0.9013157894736842, - "acc_stderr": 0.024270227737522732, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.8552631578947368, + "acc_stderr": 0.028631951845930387, + "brier_score": 0.21622829183700756, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_college_mathematics": { - "acc": 0.48, - "acc_stderr": 0.050211673156867795, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.41, + "acc_stderr": 0.049431107042371025, + "brier_score": 0.678893353888229, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_conceptual_physics": { - "acc": 0.774468085106383, - "acc_stderr": 0.02732107841738753, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.723404255319149, + "acc_stderr": 0.029241883869628817, + "brier_score": 0.3747238590806247, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_abstract_algebra": { - "acc": 0.44, - "acc_stderr": 0.04988876515698589, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.47, + "acc_stderr": 0.05016135580465919, + "brier_score": 0.6528285560015504, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_high_school_computer_science": { - "acc": 0.82, - "acc_stderr": 0.038612291966536934, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.8, + "acc_stderr": 0.04020151261036847, + "brier_score": 0.257800700904363, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_machine_learning": { "acc": 0.6160714285714286, "acc_stderr": 0.04616143075028547, - "timestamp": "2024-11-27T03-38-49.243482" + "brier_score": 0.4957008287428955, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_high_school_chemistry": { - "acc": 0.6403940886699507, - "acc_stderr": 0.03376458246509567, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.6354679802955665, + "acc_stderr": 0.0338640574606209, + "brier_score": 0.4977746266472533, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_high_school_statistics": { - "acc": 0.6481481481481481, - "acc_stderr": 0.032568505702936464, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.6620370370370371, + "acc_stderr": 0.03225941352631295, + "brier_score": 0.44800329761941105, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_elementary_mathematics": { - "acc": 0.6507936507936508, - "acc_stderr": 0.02455229220934265, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.5529100529100529, + "acc_stderr": 0.025606723995777028, + "brier_score": 0.5190951310769079, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_electrical_engineering": { - "acc": 0.8, - "acc_stderr": 0.033333333333333305, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.7586206896551724, + "acc_stderr": 0.03565998174135303, + "brier_score": 0.35460163341746903, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "mmlu_high_school_mathematics": { - "acc": 0.4666666666666667, - "acc_stderr": 0.03041771696171748, - "timestamp": "2024-11-27T03-38-49.243482" + "acc": 0.37777777777777777, + "acc_stderr": 0.029560707392465718, + "brier_score": 0.6790299812588423, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-08.159671" }, "arc_challenge": { "acc": 0.6049488054607508, diff --git a/metadata/model_scores/results_01-ai__Yi-9B.json b/metadata/model_scores/results_01-ai__Yi-9B.json index d07954f..d65ac01 100644 --- a/metadata/model_scores/results_01-ai__Yi-9B.json +++ b/metadata/model_scores/results_01-ai__Yi-9B.json @@ -1,292 +1,406 @@ { "model_name": "01-ai__Yi-9B", - "last_updated": "2024-12-04 11:23:34.423788", + "last_updated": "2024-12-19 13:39:00.472820", "results": { "harness": { "mmlu_world_religions": { - "acc": 0.8421052631578947, - "acc_stderr": 0.027966785859160865, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.8245614035087719, + "acc_stderr": 0.02917088550072766, + "brier_score": 0.2393628034564287, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_formal_logic": { - "acc": 0.5793650793650794, - "acc_stderr": 0.04415438226743745, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.5873015873015873, + "acc_stderr": 0.04403438954768177, + "brier_score": 0.5190625827333886, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_prehistory": { - "acc": 0.7716049382716049, - "acc_stderr": 0.023358211840626267, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.7901234567901234, + "acc_stderr": 0.022658344085981375, + "brier_score": 0.31645486053128496, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_moral_scenarios": { - "acc": 0.39553072625698327, - "acc_stderr": 0.016353415410075768, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.3307262569832402, + "acc_stderr": 0.01573502625896612, + "brier_score": 0.7522794482381115, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_high_school_world_history": { - "acc": 0.8227848101265823, - "acc_stderr": 0.024856364184503224, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.7974683544303798, + "acc_stderr": 0.02616056824660146, + "brier_score": 0.2508888327661202, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_moral_disputes": { - "acc": 0.7658959537572254, - "acc_stderr": 0.022797110278071128, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.7543352601156069, + "acc_stderr": 0.023176298203992005, + "brier_score": 0.35635074208151685, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_professional_law": { - "acc": 0.4941329856584094, - "acc_stderr": 0.012769356925216526, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.5097783572359843, + "acc_stderr": 0.012767793787729345, + "brier_score": 0.6218945900217582, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_logical_fallacies": { - "acc": 0.7668711656441718, - "acc_stderr": 0.033220157957767414, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.8159509202453987, + "acc_stderr": 0.030446777687971733, + "brier_score": 0.244189973620685, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_high_school_us_history": { - "acc": 0.8676470588235294, - "acc_stderr": 0.023784297520918856, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.8431372549019608, + "acc_stderr": 0.025524722324553332, + "brier_score": 0.21478127586241644, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_philosophy": { - "acc": 0.77491961414791, - "acc_stderr": 0.023720088516179034, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.7652733118971061, + "acc_stderr": 0.02407180588767703, + "brier_score": 0.3225904300433902, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_jurisprudence": { "acc": 0.7870370370370371, - "acc_stderr": 0.039578354719809784, - "timestamp": "2024-11-27T11-17-20.652776" + "acc_stderr": 0.03957835471980979, + "brier_score": 0.3012512676050204, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_international_law": { - "acc": 0.8264462809917356, - "acc_stderr": 0.0345727283691767, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.7933884297520661, + "acc_stderr": 0.03695980128098824, + "brier_score": 0.29858442209922126, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_high_school_european_history": { - "acc": 0.7696969696969697, - "acc_stderr": 0.0328766675860349, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.8121212121212121, + "acc_stderr": 0.03050193405942914, + "brier_score": 0.29980546564414023, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_high_school_government_and_politics": { - "acc": 0.9326424870466321, - "acc_stderr": 0.018088393839078912, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.9015544041450777, + "acc_stderr": 0.021500249576033456, + "brier_score": 0.149703049985458, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_high_school_microeconomics": { - "acc": 0.8235294117647058, - "acc_stderr": 0.024762902678057926, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.8361344537815126, + "acc_stderr": 0.024044054940440488, + "brier_score": 0.24282925696619972, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_high_school_geography": { - "acc": 0.8535353535353535, - "acc_stderr": 0.02519092111460393, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.8434343434343434, + "acc_stderr": 0.025890520358141454, + "brier_score": 0.20090861352016387, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_high_school_psychology": { - "acc": 0.8660550458715597, - "acc_stderr": 0.014602811435592635, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.8697247706422019, + "acc_stderr": 0.014431862852473243, + "brier_score": 0.19285202464341403, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_public_relations": { - "acc": 0.7363636363636363, - "acc_stderr": 0.04220224692971987, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.6909090909090909, + "acc_stderr": 0.044262946482000985, + "brier_score": 0.40149977698047357, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_us_foreign_policy": { - "acc": 0.9, - "acc_stderr": 0.030151134457776348, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.87, + "acc_stderr": 0.03379976689896309, + "brier_score": 0.164017036983503, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_sociology": { - "acc": 0.8606965174129353, - "acc_stderr": 0.02448448716291397, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.8557213930348259, + "acc_stderr": 0.02484575321230605, + "brier_score": 0.2072068233405296, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_high_school_macroeconomics": { - "acc": 0.764102564102564, - "acc_stderr": 0.021525965407408726, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.735897435897436, + "acc_stderr": 0.022352193737453275, + "brier_score": 0.3394529373525978, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_security_studies": { - "acc": 0.763265306122449, - "acc_stderr": 0.02721283588407314, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.6938775510204082, + "acc_stderr": 0.02950489645459597, + "brier_score": 0.3938163213040265, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_professional_psychology": { - "acc": 0.7058823529411765, - "acc_stderr": 0.018433427649401896, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.7140522875816994, + "acc_stderr": 0.01828048507295467, + "brier_score": 0.3968097961640215, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_human_sexuality": { - "acc": 0.7862595419847328, - "acc_stderr": 0.0359546161177469, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.7633587786259542, + "acc_stderr": 0.03727673575596914, + "brier_score": 0.3032759176270539, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_econometrics": { "acc": 0.5263157894736842, "acc_stderr": 0.046970851366478626, - "timestamp": "2024-11-27T11-17-20.652776" + "brier_score": 0.6006660857847085, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_miscellaneous": { - "acc": 0.8390804597701149, - "acc_stderr": 0.01314022551561173, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.8454661558109834, + "acc_stderr": 0.01292577349509594, + "brier_score": 0.2137624502707027, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_marketing": { - "acc": 0.905982905982906, - "acc_stderr": 0.019119892798924974, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.8974358974358975, + "acc_stderr": 0.019875655027867457, + "brier_score": 0.15446146980677683, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_management": { - "acc": 0.8252427184466019, - "acc_stderr": 0.03760178006026621, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.8058252427184466, + "acc_stderr": 0.039166677628225836, + "brier_score": 0.24991590987005177, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_nutrition": { - "acc": 0.7549019607843137, - "acc_stderr": 0.02463004897982476, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.7418300653594772, + "acc_stderr": 0.02505850331695815, + "brier_score": 0.35022594488486103, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_medical_genetics": { - "acc": 0.75, - "acc_stderr": 0.04351941398892446, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.79, + "acc_stderr": 0.040936018074033256, + "brier_score": 0.27818260990303734, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_human_aging": { - "acc": 0.7488789237668162, - "acc_stderr": 0.02910522083322462, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.7354260089686099, + "acc_stderr": 0.02960510321703833, + "brier_score": 0.3703005196752836, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_professional_medicine": { - "acc": 0.7058823529411765, - "acc_stderr": 0.0276784686421447, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.7022058823529411, + "acc_stderr": 0.027778298701545443, + "brier_score": 0.418253524032865, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_college_medicine": { - "acc": 0.7283236994219653, - "acc_stderr": 0.0339175032232166, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.7167630057803468, + "acc_stderr": 0.034355680560478746, + "brier_score": 0.3778158592653711, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_business_ethics": { - "acc": 0.79, - "acc_stderr": 0.040936018074033256, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.76, + "acc_stderr": 0.04292346959909281, + "brier_score": 0.26292444484546645, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_clinical_knowledge": { "acc": 0.7283018867924528, - "acc_stderr": 0.027377706624670713, - "timestamp": "2024-11-27T11-17-20.652776" + "acc_stderr": 0.02737770662467071, + "brier_score": 0.3504868520343354, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_global_facts": { - "acc": 0.34, - "acc_stderr": 0.04760952285695235, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.42, + "acc_stderr": 0.049604496374885836, + "brier_score": 0.6381647082307358, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_virology": { - "acc": 0.5060240963855421, - "acc_stderr": 0.03892212195333045, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.5120481927710844, + "acc_stderr": 0.03891364495835817, + "brier_score": 0.6876721600653987, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_professional_accounting": { - "acc": 0.5602836879432624, - "acc_stderr": 0.029609912075594106, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.5425531914893617, + "acc_stderr": 0.029719281272236855, + "brier_score": 0.5609721539756205, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_college_physics": { - "acc": 0.46078431372549017, - "acc_stderr": 0.049598599663841815, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.45098039215686275, + "acc_stderr": 0.04951218252396262, + "brier_score": 0.567059537923981, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_high_school_physics": { - "acc": 0.4105960264900662, - "acc_stderr": 0.04016689594849929, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.47019867549668876, + "acc_stderr": 0.040752249922169775, + "brier_score": 0.6578173971603419, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_high_school_biology": { - "acc": 0.8483870967741935, - "acc_stderr": 0.02040261665441674, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.8354838709677419, + "acc_stderr": 0.021090847745939324, + "brier_score": 0.22862158813414019, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_college_biology": { - "acc": 0.8263888888888888, - "acc_stderr": 0.03167473383795718, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.8194444444444444, + "acc_stderr": 0.032166008088022675, + "brier_score": 0.2435335430771714, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_anatomy": { - "acc": 0.5851851851851851, - "acc_stderr": 0.04256193767901408, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.6296296296296297, + "acc_stderr": 0.04171654161354543, + "brier_score": 0.47050823290456106, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_college_chemistry": { - "acc": 0.53, - "acc_stderr": 0.050161355804659205, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.48, + "acc_stderr": 0.050211673156867795, + "brier_score": 0.6012524246891047, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_computer_security": { - "acc": 0.82, - "acc_stderr": 0.038612291966536934, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.83, + "acc_stderr": 0.03775251680686371, + "brier_score": 0.2578662140207141, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_college_computer_science": { - "acc": 0.61, - "acc_stderr": 0.04902071300001974, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.66, + "acc_stderr": 0.04760952285695237, + "brier_score": 0.4716753520570882, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_astronomy": { - "acc": 0.756578947368421, - "acc_stderr": 0.034923496688842384, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.7368421052631579, + "acc_stderr": 0.035834961763610736, + "brier_score": 0.34869431661231887, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_college_mathematics": { - "acc": 0.44, - "acc_stderr": 0.049888765156985884, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.47, + "acc_stderr": 0.050161355804659205, + "brier_score": 0.6669320434689762, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_conceptual_physics": { - "acc": 0.7106382978723405, - "acc_stderr": 0.02964400657700962, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.7319148936170212, + "acc_stderr": 0.028957342788342347, + "brier_score": 0.36516100066393176, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_abstract_algebra": { - "acc": 0.3, - "acc_stderr": 0.046056618647183814, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.37, + "acc_stderr": 0.048523658709391, + "brier_score": 0.7279423806334974, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_high_school_computer_science": { - "acc": 0.85, - "acc_stderr": 0.035887028128263686, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.88, + "acc_stderr": 0.03265986323710905, + "brier_score": 0.22146883789630473, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_machine_learning": { - "acc": 0.5714285714285714, - "acc_stderr": 0.04697113923010212, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.5982142857142857, + "acc_stderr": 0.04653333146973647, + "brier_score": 0.5179734588074891, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_high_school_chemistry": { - "acc": 0.5862068965517241, - "acc_stderr": 0.03465304488406796, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.6059113300492611, + "acc_stderr": 0.03438157967036545, + "brier_score": 0.4971528583419886, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_high_school_statistics": { - "acc": 0.6805555555555556, - "acc_stderr": 0.03179876342176851, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.6712962962962963, + "acc_stderr": 0.032036140846700596, + "brier_score": 0.47464895250991695, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_elementary_mathematics": { - "acc": 0.5978835978835979, - "acc_stderr": 0.02525303255499769, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.6137566137566137, + "acc_stderr": 0.025075981767601677, + "brier_score": 0.48844536148725254, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_electrical_engineering": { - "acc": 0.7310344827586207, - "acc_stderr": 0.03695183311650232, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.7034482758620689, + "acc_stderr": 0.03806142687309992, + "brier_score": 0.4060122900878722, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "mmlu_high_school_mathematics": { - "acc": 0.40370370370370373, - "acc_stderr": 0.029914812342227627, - "timestamp": "2024-11-27T11-17-20.652776" + "acc": 0.4185185185185185, + "acc_stderr": 0.03007801307502206, + "brier_score": 0.6716508120089633, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-04-18.966055" }, "arc_challenge": { "acc": 0.5708191126279863, diff --git a/metadata/model_scores/results_EleutherAI__gpt-j-6b.json b/metadata/model_scores/results_EleutherAI__gpt-j-6b.json index 1e0603a..f3a1397 100644 --- a/metadata/model_scores/results_EleutherAI__gpt-j-6b.json +++ b/metadata/model_scores/results_EleutherAI__gpt-j-6b.json @@ -1,6 +1,6 @@ { "model_name": "EleutherAI/gpt-j-6b", - "last_updated": "2024-12-04 11:26:02.029746", + "last_updated": "2024-12-19 13:42:20.535804", "results": { "harness": { "minerva_math_precalc": { @@ -228,289 +228,403 @@ "timestamp": "2024-06-13T17-26-24.151637" }, "mmlu_world_religions": { - "acc": 0.34502923976608185, - "acc_stderr": 0.036459813773888065, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.3157894736842105, + "acc_stderr": 0.03565079670708311, + "brier_score": 0.7374101068019413, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_formal_logic": { - "acc": 0.18253968253968253, - "acc_stderr": 0.03455071019102151, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.25396825396825395, + "acc_stderr": 0.03893259610604674, + "brier_score": 0.7611818191664562, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_prehistory": { - "acc": 0.30864197530864196, - "acc_stderr": 0.02570264026060376, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.3148148148148148, + "acc_stderr": 0.02584224870090217, + "brier_score": 0.7444675213456617, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_moral_scenarios": { "acc": 0.2424581005586592, - "acc_stderr": 0.014333522059217892, - "timestamp": "2024-11-22T19-43-59.503231" + "acc_stderr": 0.014333522059217887, + "brier_score": 0.7653061951507586, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_high_school_world_history": { - "acc": 0.2869198312236287, - "acc_stderr": 0.029443773022594693, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.26582278481012656, + "acc_stderr": 0.028756799629658332, + "brier_score": 0.7500397365242264, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_moral_disputes": { - "acc": 0.26878612716763006, - "acc_stderr": 0.02386800326250011, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.28901734104046245, + "acc_stderr": 0.02440517393578323, + "brier_score": 0.7481647628004192, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_professional_law": { - "acc": 0.288135593220339, - "acc_stderr": 0.011567140661324568, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.26010430247718386, + "acc_stderr": 0.011204382887823834, + "brier_score": 0.7520231732408674, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_logical_fallacies": { - "acc": 0.24539877300613497, - "acc_stderr": 0.03380939813943353, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.2147239263803681, + "acc_stderr": 0.03226219377286774, + "brier_score": 0.7640208655687383, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_high_school_us_history": { - "acc": 0.28431372549019607, - "acc_stderr": 0.03166009679399813, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.24019607843137256, + "acc_stderr": 0.02998373305591361, + "brier_score": 0.7541644655494616, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_philosophy": { - "acc": 0.26688102893890675, - "acc_stderr": 0.02512263760881664, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.24437299035369775, + "acc_stderr": 0.024406162094668907, + "brier_score": 0.7488388338331259, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_jurisprudence": { - "acc": 0.26851851851851855, - "acc_stderr": 0.04284467968052191, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.28703703703703703, + "acc_stderr": 0.043733130409147614, + "brier_score": 0.7477899989019611, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_international_law": { "acc": 0.24793388429752067, - "acc_stderr": 0.03941897526516303, - "timestamp": "2024-11-22T19-43-59.503231" + "acc_stderr": 0.039418975265163025, + "brier_score": 0.7521289705486953, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_high_school_european_history": { - "acc": 0.2787878787878788, - "acc_stderr": 0.035014387062967806, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.19393939393939394, + "acc_stderr": 0.030874145136562114, + "brier_score": 0.7651629152934769, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_high_school_government_and_politics": { - "acc": 0.21761658031088082, - "acc_stderr": 0.02977866303775296, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.25906735751295334, + "acc_stderr": 0.031618779179354094, + "brier_score": 0.7461302893750941, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_high_school_microeconomics": { - "acc": 0.2647058823529412, - "acc_stderr": 0.02865749128507196, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.3277310924369748, + "acc_stderr": 0.03048991141767323, + "brier_score": 0.7383799741012769, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_high_school_geography": { - "acc": 0.23232323232323232, - "acc_stderr": 0.030088629490217483, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.21717171717171718, + "acc_stderr": 0.02937661648494563, + "brier_score": 0.7626662681118054, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_high_school_psychology": { - "acc": 0.23302752293577983, - "acc_stderr": 0.018125669180861507, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.23669724770642203, + "acc_stderr": 0.01822407811729908, + "brier_score": 0.7602229296110636, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_public_relations": { - "acc": 0.34545454545454546, - "acc_stderr": 0.04554619617541054, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.2727272727272727, + "acc_stderr": 0.04265792110940588, + "brier_score": 0.7399827596672014, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_us_foreign_policy": { - "acc": 0.29, - "acc_stderr": 0.045604802157206845, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.38, + "acc_stderr": 0.048783173121456316, + "brier_score": 0.731017853803002, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_sociology": { - "acc": 0.2885572139303483, - "acc_stderr": 0.03203841040213322, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.2835820895522388, + "acc_stderr": 0.03187187537919796, + "brier_score": 0.7487866256318877, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_high_school_macroeconomics": { - "acc": 0.2358974358974359, - "acc_stderr": 0.021525965407408726, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.2923076923076923, + "acc_stderr": 0.023060438380857744, + "brier_score": 0.7432336631802331, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_security_studies": { - "acc": 0.35918367346938773, - "acc_stderr": 0.03071356045510849, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.32653061224489793, + "acc_stderr": 0.030021056238440296, + "brier_score": 0.7326057170381776, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_professional_psychology": { - "acc": 0.27941176470588236, - "acc_stderr": 0.018152871051538826, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.2581699346405229, + "acc_stderr": 0.017704531653250075, + "brier_score": 0.7533792173986266, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_human_sexuality": { - "acc": 0.22137404580152673, - "acc_stderr": 0.03641297081313729, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.2900763358778626, + "acc_stderr": 0.03980066246467766, + "brier_score": 0.7356382300150744, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_econometrics": { - "acc": 0.2807017543859649, - "acc_stderr": 0.042270544512322, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.20175438596491227, + "acc_stderr": 0.037752050135836386, + "brier_score": 0.7629119690975171, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_miscellaneous": { - "acc": 0.3128991060025543, - "acc_stderr": 0.016580935940304055, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.28607918263090676, + "acc_stderr": 0.016160871405127543, + "brier_score": 0.7480858931743369, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_marketing": { - "acc": 0.27350427350427353, - "acc_stderr": 0.02920254015343117, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.2905982905982906, + "acc_stderr": 0.029745048572674054, + "brier_score": 0.74792602280863, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_management": { - "acc": 0.20388349514563106, - "acc_stderr": 0.039891398595317706, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.23300970873786409, + "acc_stderr": 0.04185832598928315, + "brier_score": 0.7458389327916556, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_nutrition": { - "acc": 0.2647058823529412, - "acc_stderr": 0.025261691219729494, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.3202614379084967, + "acc_stderr": 0.02671611838015684, + "brier_score": 0.739306232085926, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_medical_genetics": { - "acc": 0.29, - "acc_stderr": 0.04560480215720684, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.34, + "acc_stderr": 0.04760952285695235, + "brier_score": 0.7314187897221344, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_human_aging": { - "acc": 0.3273542600896861, - "acc_stderr": 0.03149384670994131, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.38565022421524664, + "acc_stderr": 0.03266842214289201, + "brier_score": 0.7342953572667033, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_professional_medicine": { - "acc": 0.23897058823529413, - "acc_stderr": 0.025905280644893006, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.20955882352941177, + "acc_stderr": 0.024723110407677083, + "brier_score": 0.8002697306213429, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_college_medicine": { - "acc": 0.27167630057803466, - "acc_stderr": 0.03391750322321659, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.2254335260115607, + "acc_stderr": 0.03186209851641144, + "brier_score": 0.7592946121995308, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_business_ethics": { - "acc": 0.29, - "acc_stderr": 0.045604802157206845, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.32, + "acc_stderr": 0.046882617226215034, + "brier_score": 0.7491275586376847, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_clinical_knowledge": { - "acc": 0.25660377358490566, - "acc_stderr": 0.02688064788905199, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.24150943396226415, + "acc_stderr": 0.026341480371118355, + "brier_score": 0.7553629314392128, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_global_facts": { - "acc": 0.22, - "acc_stderr": 0.0416333199893227, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.36, + "acc_stderr": 0.048241815132442176, + "brier_score": 0.7304686187312108, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_virology": { - "acc": 0.3373493975903614, - "acc_stderr": 0.03680783690727581, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.3433734939759036, + "acc_stderr": 0.03696584317010601, + "brier_score": 0.7374895776286096, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_professional_accounting": { - "acc": 0.2765957446808511, - "acc_stderr": 0.026684564340461025, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.28368794326241137, + "acc_stderr": 0.02689170942834396, + "brier_score": 0.7505336664138332, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_college_physics": { - "acc": 0.21568627450980393, - "acc_stderr": 0.04092563958237654, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.18627450980392157, + "acc_stderr": 0.03873958714149353, + "brier_score": 0.7800486202461536, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_high_school_physics": { - "acc": 0.26490066225165565, - "acc_stderr": 0.03603038545360384, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.2119205298013245, + "acc_stderr": 0.03336767086567977, + "brier_score": 0.7503552732060065, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_high_school_biology": { - "acc": 0.20967741935483872, - "acc_stderr": 0.023157879349083515, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.25483870967741934, + "acc_stderr": 0.02479011845933221, + "brier_score": 0.7596491887421389, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_college_biology": { - "acc": 0.25, - "acc_stderr": 0.03621034121889507, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.22916666666666666, + "acc_stderr": 0.035146974678623884, + "brier_score": 0.7588401240333378, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_anatomy": { - "acc": 0.2740740740740741, - "acc_stderr": 0.03853254836552003, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.24444444444444444, + "acc_stderr": 0.037125378336148665, + "brier_score": 0.7514346773938658, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_college_chemistry": { - "acc": 0.18, - "acc_stderr": 0.03861229196653694, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 0.7555281959150324, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_computer_security": { - "acc": 0.41, - "acc_stderr": 0.049431107042371025, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "brier_score": 0.7391551892716636, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_college_computer_science": { - "acc": 0.23, - "acc_stderr": 0.042295258468165044, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.27, + "acc_stderr": 0.0446196043338474, + "brier_score": 0.7604929068028424, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_astronomy": { - "acc": 0.2631578947368421, - "acc_stderr": 0.03583496176361064, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.2894736842105263, + "acc_stderr": 0.03690677986137283, + "brier_score": 0.7449180437327387, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_college_mathematics": { - "acc": 0.31, - "acc_stderr": 0.04648231987117316, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "brier_score": 0.754253155280133, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_conceptual_physics": { - "acc": 0.33617021276595743, - "acc_stderr": 0.030881618520676942, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.30638297872340425, + "acc_stderr": 0.03013590647851756, + "brier_score": 0.7425276218862534, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_abstract_algebra": { "acc": 0.29, - "acc_stderr": 0.045604802157206845, - "timestamp": "2024-11-22T19-43-59.503231" + "acc_stderr": 0.04560480215720683, + "brier_score": 0.755635744034552, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_high_school_computer_science": { - "acc": 0.15, - "acc_stderr": 0.03588702812826373, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.22, + "acc_stderr": 0.0416333199893227, + "brier_score": 0.7494321719105557, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_machine_learning": { - "acc": 0.38392857142857145, - "acc_stderr": 0.04616143075028547, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.32142857142857145, + "acc_stderr": 0.0443280405529152, + "brier_score": 0.7430945873962429, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_high_school_chemistry": { - "acc": 0.24630541871921183, - "acc_stderr": 0.03031509928561773, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.270935960591133, + "acc_stderr": 0.031270907132976984, + "brier_score": 0.753668782444299, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_high_school_statistics": { - "acc": 0.1574074074074074, - "acc_stderr": 0.02483717351824239, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.19444444444444445, + "acc_stderr": 0.026991454502036726, + "brier_score": 0.7804393461721738, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_elementary_mathematics": { - "acc": 0.23544973544973544, - "acc_stderr": 0.021851509822031708, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.2619047619047619, + "acc_stderr": 0.022644212615525218, + "brier_score": 0.7590777292193703, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_electrical_engineering": { - "acc": 0.2896551724137931, - "acc_stderr": 0.037800192304380135, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.3448275862068966, + "acc_stderr": 0.039609335494512087, + "brier_score": 0.7334444866577424, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "mmlu_high_school_mathematics": { - "acc": 0.2851851851851852, - "acc_stderr": 0.027528599210340492, - "timestamp": "2024-11-22T19-43-59.503231" + "acc": 0.25925925925925924, + "acc_stderr": 0.02671924078371217, + "brier_score": 0.757412220897825, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-46-36.775995" }, "arc_challenge": { "acc": 0.36177474402730375, diff --git a/metadata/model_scores/results_EleutherAI__gpt-neo-2.7B.json b/metadata/model_scores/results_EleutherAI__gpt-neo-2.7B.json index 54e567d..f526af1 100644 --- a/metadata/model_scores/results_EleutherAI__gpt-neo-2.7B.json +++ b/metadata/model_scores/results_EleutherAI__gpt-neo-2.7B.json @@ -1,6 +1,6 @@ { "model_name": "EleutherAI/gpt-neo-2.7B", - "last_updated": "2024-12-04 11:24:49.074976", + "last_updated": "2024-12-19 13:40:46.199004", "results": { "harness": { "minerva_math_precalc": { @@ -228,289 +228,403 @@ "timestamp": "2024-06-14T08-32-02.667192" }, "mmlu_world_religions": { - "acc": 0.27485380116959063, - "acc_stderr": 0.03424042924691584, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.2631578947368421, + "acc_stderr": 0.03377310252209197, + "brier_score": 0.7658346958459136, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_formal_logic": { - "acc": 0.18253968253968253, - "acc_stderr": 0.03455071019102149, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.2222222222222222, + "acc_stderr": 0.037184890068181146, + "brier_score": 0.7794135178208829, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_prehistory": { - "acc": 0.3117283950617284, - "acc_stderr": 0.02577311116963045, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.24382716049382716, + "acc_stderr": 0.023891879541959614, + "brier_score": 0.7603184815640671, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_moral_scenarios": { - "acc": 0.27039106145251396, - "acc_stderr": 0.014854993938010085, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.27262569832402234, + "acc_stderr": 0.014893391735249603, + "brier_score": 0.750880388733267, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_high_school_world_history": { - "acc": 0.22362869198312235, - "acc_stderr": 0.02712329820522997, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.23628691983122363, + "acc_stderr": 0.027652153144159274, + "brier_score": 0.752884223832844, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_moral_disputes": { - "acc": 0.24566473988439305, - "acc_stderr": 0.023176298203992005, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.26011560693641617, + "acc_stderr": 0.023618678310069367, + "brier_score": 0.7615525596388655, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_professional_law": { - "acc": 0.2457627118644068, - "acc_stderr": 0.01099615663514269, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.23598435462842243, + "acc_stderr": 0.010844802669662694, + "brier_score": 0.7670687906909094, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_logical_fallacies": { - "acc": 0.25766871165644173, - "acc_stderr": 0.03436150827846917, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.2883435582822086, + "acc_stderr": 0.035590395316173425, + "brier_score": 0.7539889704237094, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_high_school_us_history": { - "acc": 0.19607843137254902, - "acc_stderr": 0.027865942286639318, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.28431372549019607, + "acc_stderr": 0.03166009679399812, + "brier_score": 0.7625875980655249, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_philosophy": { - "acc": 0.3247588424437299, - "acc_stderr": 0.026596782287697046, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.20257234726688103, + "acc_stderr": 0.02282731749105968, + "brier_score": 0.778702284090794, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_jurisprudence": { - "acc": 0.2962962962962963, - "acc_stderr": 0.04414343666854933, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.25925925925925924, + "acc_stderr": 0.042365112580946336, + "brier_score": 0.759013828336546, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_international_law": { - "acc": 0.23140495867768596, - "acc_stderr": 0.03849856098794088, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.256198347107438, + "acc_stderr": 0.03984979653302871, + "brier_score": 0.7520643776248898, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_high_school_european_history": { - "acc": 0.23030303030303031, - "acc_stderr": 0.0328766675860349, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.18787878787878787, + "acc_stderr": 0.030501934059429144, + "brier_score": 0.7758275559833471, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_high_school_government_and_politics": { - "acc": 0.26424870466321243, - "acc_stderr": 0.03182155050916646, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.21243523316062177, + "acc_stderr": 0.029519282616817258, + "brier_score": 0.7830004814871769, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_high_school_microeconomics": { - "acc": 0.23529411764705882, - "acc_stderr": 0.027553614467863804, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.24369747899159663, + "acc_stderr": 0.027886828078380565, + "brier_score": 0.7596873547074825, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_high_school_geography": { - "acc": 0.3282828282828283, - "acc_stderr": 0.03345678422756775, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.25252525252525254, + "acc_stderr": 0.030954055470365907, + "brier_score": 0.7673749782882908, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_high_school_psychology": { - "acc": 0.30642201834862387, - "acc_stderr": 0.01976551722045852, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.24036697247706423, + "acc_stderr": 0.01832060732096407, + "brier_score": 0.7678113442767674, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_public_relations": { - "acc": 0.18181818181818182, - "acc_stderr": 0.036942843353378, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.2909090909090909, + "acc_stderr": 0.04350271442923243, + "brier_score": 0.7594777018665088, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_us_foreign_policy": { - "acc": 0.29, - "acc_stderr": 0.045604802157206845, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 0.7619763802399224, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_sociology": { - "acc": 0.22885572139303484, - "acc_stderr": 0.02970528405677243, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.263681592039801, + "acc_stderr": 0.03115715086935557, + "brier_score": 0.7685246417582672, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_high_school_macroeconomics": { - "acc": 0.3487179487179487, - "acc_stderr": 0.024162780284017717, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.2205128205128205, + "acc_stderr": 0.0210206726808279, + "brier_score": 0.7621025466387151, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_security_studies": { - "acc": 0.2816326530612245, - "acc_stderr": 0.02879518557429127, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.19183673469387755, + "acc_stderr": 0.02520696315422539, + "brier_score": 0.791629502080558, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_professional_psychology": { - "acc": 0.2679738562091503, - "acc_stderr": 0.017917974069594726, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.28431372549019607, + "acc_stderr": 0.018249024411207654, + "brier_score": 0.749931090787944, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_human_sexuality": { - "acc": 0.2748091603053435, - "acc_stderr": 0.03915345408847836, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.2595419847328244, + "acc_stderr": 0.03844876139785271, + "brier_score": 0.7653318578140017, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_econometrics": { - "acc": 0.2631578947368421, - "acc_stderr": 0.04142439719489361, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.21929824561403508, + "acc_stderr": 0.03892431106518754, + "brier_score": 0.7547973992786374, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_miscellaneous": { - "acc": 0.2388250319284802, - "acc_stderr": 0.015246803197398682, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.2988505747126437, + "acc_stderr": 0.01636925681509313, + "brier_score": 0.7496301587887607, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_marketing": { - "acc": 0.2606837606837607, - "acc_stderr": 0.028760348956523414, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.20512820512820512, + "acc_stderr": 0.026453508054040346, + "brier_score": 0.7526073931650051, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_management": { - "acc": 0.27184466019417475, - "acc_stderr": 0.044052680241409216, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.17475728155339806, + "acc_stderr": 0.03760178006026621, + "brier_score": 0.7826546414599174, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_nutrition": { - "acc": 0.3104575163398693, - "acc_stderr": 0.026493033225145894, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.22875816993464052, + "acc_stderr": 0.02405102973991225, + "brier_score": 0.759395961830524, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_medical_genetics": { - "acc": 0.29, - "acc_stderr": 0.04560480215720684, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.32, + "acc_stderr": 0.046882617226215034, + "brier_score": 0.7474368668384139, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_human_aging": { - "acc": 0.18385650224215247, - "acc_stderr": 0.02599837909235651, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.24663677130044842, + "acc_stderr": 0.028930413120910877, + "brier_score": 0.745585259751547, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_professional_medicine": { - "acc": 0.43014705882352944, - "acc_stderr": 0.030074971917302875, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.19852941176470587, + "acc_stderr": 0.024231013370541073, + "brier_score": 0.7964218925708318, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_college_medicine": { - "acc": 0.23699421965317918, - "acc_stderr": 0.032424147574830996, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.19653179190751446, + "acc_stderr": 0.03029957466478814, + "brier_score": 0.7652411263996571, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_business_ethics": { - "acc": 0.29, - "acc_stderr": 0.045604802157206845, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 0.7525584188309182, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_clinical_knowledge": { - "acc": 0.2641509433962264, - "acc_stderr": 0.027134291628741713, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.2981132075471698, + "acc_stderr": 0.028152837942493857, + "brier_score": 0.7492196086319309, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_global_facts": { - "acc": 0.21, - "acc_stderr": 0.040936018074033256, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.28, + "acc_stderr": 0.04512608598542127, + "brier_score": 0.7337682181369112, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_virology": { - "acc": 0.3253012048192771, - "acc_stderr": 0.03647168523683229, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.25301204819277107, + "acc_stderr": 0.033844291552331346, + "brier_score": 0.7498117129055916, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_professional_accounting": { - "acc": 0.2553191489361702, - "acc_stderr": 0.026011992930902013, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.28368794326241137, + "acc_stderr": 0.02689170942834396, + "brier_score": 0.7608069606813758, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_college_physics": { - "acc": 0.21568627450980393, - "acc_stderr": 0.04092563958237657, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.37254901960784315, + "acc_stderr": 0.04810840148082635, + "brier_score": 0.7493495275284697, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_high_school_physics": { - "acc": 0.23841059602649006, - "acc_stderr": 0.0347918557259966, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.24503311258278146, + "acc_stderr": 0.035118075718047245, + "brier_score": 0.7608706788430074, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_high_school_biology": { - "acc": 0.24838709677419354, - "acc_stderr": 0.024580028921481003, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.22258064516129034, + "acc_stderr": 0.02366421667164252, + "brier_score": 0.7621223319968164, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_college_biology": { - "acc": 0.2569444444444444, - "acc_stderr": 0.03653946969442099, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.22916666666666666, + "acc_stderr": 0.035146974678623884, + "brier_score": 0.76157444703166, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_anatomy": { - "acc": 0.2, - "acc_stderr": 0.03455473702325436, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.31851851851851853, + "acc_stderr": 0.040247784019771096, + "brier_score": 0.7453968548735045, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_college_chemistry": { - "acc": 0.22, - "acc_stderr": 0.04163331998932269, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.26, + "acc_stderr": 0.044084400227680794, + "brier_score": 0.7698692994140525, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_computer_security": { - "acc": 0.28, - "acc_stderr": 0.045126085985421276, - "timestamp": "2024-11-21T20-01-14.372897" - }, - "mmlu_college_computer_science": { "acc": 0.25, "acc_stderr": 0.04351941398892446, - "timestamp": "2024-11-21T20-01-14.372897" + "brier_score": 0.7556765474161725, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" + }, + "mmlu_college_computer_science": { + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "brier_score": 0.7501910451686946, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_astronomy": { - "acc": 0.19078947368421054, - "acc_stderr": 0.031975658210325004, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.2236842105263158, + "acc_stderr": 0.033911609343436025, + "brier_score": 0.7611459185050019, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_college_mathematics": { - "acc": 0.28, - "acc_stderr": 0.04512608598542127, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.26, + "acc_stderr": 0.044084400227680794, + "brier_score": 0.7618346898017976, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_conceptual_physics": { - "acc": 0.2765957446808511, - "acc_stderr": 0.029241883869628817, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.22127659574468084, + "acc_stderr": 0.027136349602424063, + "brier_score": 0.7569627671463691, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_abstract_algebra": { - "acc": 0.24, - "acc_stderr": 0.042923469599092816, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 0.7624468731602159, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_high_school_computer_science": { - "acc": 0.35, - "acc_stderr": 0.0479372485441102, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.29, + "acc_stderr": 0.04560480215720684, + "brier_score": 0.755932464895756, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_machine_learning": { - "acc": 0.16964285714285715, - "acc_stderr": 0.0356236785009539, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.22321428571428573, + "acc_stderr": 0.03952301967702511, + "brier_score": 0.7564926169126464, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_high_school_chemistry": { - "acc": 0.2413793103448276, - "acc_stderr": 0.030108330718011625, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.26108374384236455, + "acc_stderr": 0.030903796952114503, + "brier_score": 0.7478953532965273, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_high_school_statistics": { - "acc": 0.4074074074074074, - "acc_stderr": 0.03350991604696043, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.1712962962962963, + "acc_stderr": 0.025695341643824688, + "brier_score": 0.7791131334160182, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_elementary_mathematics": { - "acc": 0.29365079365079366, - "acc_stderr": 0.02345603738398203, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.25396825396825395, + "acc_stderr": 0.022418042891113942, + "brier_score": 0.7597662971310662, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_electrical_engineering": { - "acc": 0.25517241379310346, - "acc_stderr": 0.03632984052707842, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.2206896551724138, + "acc_stderr": 0.03455930201924811, + "brier_score": 0.7641719279038703, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "mmlu_high_school_mathematics": { - "acc": 0.24814814814814815, - "acc_stderr": 0.0263357394040558, - "timestamp": "2024-11-21T20-01-14.372897" + "acc": 0.2851851851851852, + "acc_stderr": 0.027528599210340492, + "brier_score": 0.7541280656672071, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-09-00.779237" }, "arc_challenge": { "acc": 0.3199658703071672, diff --git a/metadata/model_scores/results_EleutherAI__pile-t5-large.json b/metadata/model_scores/results_EleutherAI__pile-t5-large.json index f8323ec..87eaf79 100644 --- a/metadata/model_scores/results_EleutherAI__pile-t5-large.json +++ b/metadata/model_scores/results_EleutherAI__pile-t5-large.json @@ -1,292 +1,406 @@ { "model_name": "EleutherAI__pile-t5-large", - "last_updated": "2024-12-04 11:22:27.933367", + "last_updated": "2024-12-19 13:37:43.594027", "results": { "harness": { "mmlu_world_religions": { "acc": 0.3216374269005848, "acc_stderr": 0.03582529442573122, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.7849235689850791, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_formal_logic": { "acc": 0.2857142857142857, "acc_stderr": 0.04040610178208841, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.7491823679893918, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_prehistory": { "acc": 0.21604938271604937, "acc_stderr": 0.022899162918445813, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.8190683081054467, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_moral_scenarios": { "acc": 0.23798882681564246, "acc_stderr": 0.014242630070574885, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.797061418262279, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_high_school_world_history": { "acc": 0.270042194092827, "acc_stderr": 0.028900721906293426, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.7967472738757885, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_moral_disputes": { "acc": 0.24855491329479767, "acc_stderr": 0.023267528432100174, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.8088215112211845, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_professional_law": { "acc": 0.2457627118644068, "acc_stderr": 0.01099615663514269, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.8033184156207308, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_logical_fallacies": { "acc": 0.22085889570552147, "acc_stderr": 0.032591773927421776, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.8228830625306162, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_high_school_us_history": { "acc": 0.25, "acc_stderr": 0.03039153369274154, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.7975461325721622, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_philosophy": { "acc": 0.1864951768488746, "acc_stderr": 0.02212243977248077, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.8335500054495456, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_jurisprudence": { "acc": 0.25925925925925924, "acc_stderr": 0.04236511258094634, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.7945188405927434, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_international_law": { "acc": 0.2396694214876033, "acc_stderr": 0.03896878985070417, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.8384299771362596, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_high_school_european_history": { "acc": 0.21818181818181817, "acc_stderr": 0.03225078108306289, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.8130915681740868, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_high_school_government_and_politics": { "acc": 0.19689119170984457, "acc_stderr": 0.02869787397186069, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.795124583158224, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_high_school_microeconomics": { "acc": 0.21008403361344538, "acc_stderr": 0.026461398717471874, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.7914885117838936, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_high_school_geography": { "acc": 0.17676767676767677, "acc_stderr": 0.027178752639044915, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.8114916043453221, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_high_school_psychology": { "acc": 0.1926605504587156, "acc_stderr": 0.016909276884936073, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.7989489616232881, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_public_relations": { "acc": 0.21818181818181817, "acc_stderr": 0.03955932861795833, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.8016038933957079, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_us_foreign_policy": { "acc": 0.28, "acc_stderr": 0.045126085985421276, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.7842622657608993, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_sociology": { "acc": 0.24378109452736318, "acc_stderr": 0.030360490154014652, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.7907654428049605, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_high_school_macroeconomics": { "acc": 0.20256410256410257, "acc_stderr": 0.020377660970371397, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.7907203661201027, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_security_studies": { "acc": 0.18775510204081633, "acc_stderr": 0.02500025603954622, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.8055882511825393, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_professional_psychology": { "acc": 0.25, "acc_stderr": 0.01751781884501444, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.8049013750733225, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_human_sexuality": { "acc": 0.2595419847328244, "acc_stderr": 0.03844876139785271, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.7874984574589324, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_econometrics": { "acc": 0.23684210526315788, "acc_stderr": 0.039994238792813386, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.8053528746343667, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_miscellaneous": { "acc": 0.23754789272030652, "acc_stderr": 0.015218733046150195, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.8070587651241263, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_marketing": { "acc": 0.2905982905982906, "acc_stderr": 0.029745048572674057, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.7857263845235076, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_management": { "acc": 0.17475728155339806, "acc_stderr": 0.03760178006026621, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.7993902273736592, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_nutrition": { "acc": 0.22549019607843138, "acc_stderr": 0.023929155517351284, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.7986515847391095, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_medical_genetics": { "acc": 0.3, "acc_stderr": 0.046056618647183814, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.7687063588030674, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_human_aging": { "acc": 0.31390134529147984, "acc_stderr": 0.03114679648297246, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.7821696951893335, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_professional_medicine": { "acc": 0.18382352941176472, "acc_stderr": 0.02352924218519311, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.7750922932266949, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_college_medicine": { "acc": 0.20809248554913296, "acc_stderr": 0.030952890217749884, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.8012852650549208, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_business_ethics": { "acc": 0.3, "acc_stderr": 0.046056618647183814, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.7751088518970387, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_clinical_knowledge": { "acc": 0.21509433962264152, "acc_stderr": 0.025288394502891377, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.7929367060705027, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_global_facts": { "acc": 0.18, "acc_stderr": 0.038612291966536955, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.8352222953309943, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_virology": { "acc": 0.28313253012048195, "acc_stderr": 0.03507295431370518, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.777092069370033, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_professional_accounting": { "acc": 0.23404255319148937, "acc_stderr": 0.025257861359432407, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.8063232224079224, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_college_physics": { "acc": 0.21568627450980393, "acc_stderr": 0.040925639582376556, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.7824028629646136, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_high_school_physics": { "acc": 0.1986754966887417, "acc_stderr": 0.032578473844367746, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.8110622864067462, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_high_school_biology": { "acc": 0.1774193548387097, "acc_stderr": 0.021732540689329265, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.8133831842279677, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_college_biology": { "acc": 0.2569444444444444, "acc_stderr": 0.03653946969442099, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.7892509277283707, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_anatomy": { "acc": 0.18518518518518517, "acc_stderr": 0.03355677216313142, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.8433576838154646, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_college_chemistry": { "acc": 0.2, "acc_stderr": 0.040201512610368445, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.7826406694386647, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_computer_security": { "acc": 0.28, "acc_stderr": 0.045126085985421276, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.7993730176508791, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_college_computer_science": { "acc": 0.26, "acc_stderr": 0.044084400227680794, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.7833401061895657, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_astronomy": { "acc": 0.17763157894736842, "acc_stderr": 0.031103182383123398, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.830623772201601, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_college_mathematics": { "acc": 0.21, "acc_stderr": 0.040936018074033256, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.8015392573716095, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_conceptual_physics": { "acc": 0.26382978723404255, "acc_stderr": 0.02880998985410298, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.7864530034390211, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_abstract_algebra": { "acc": 0.22, "acc_stderr": 0.04163331998932269, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.8242445987130559, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_high_school_computer_science": { "acc": 0.25, "acc_stderr": 0.04351941398892446, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.8195282392307509, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_machine_learning": { "acc": 0.3125, "acc_stderr": 0.043994650575715215, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.7781633201123034, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_high_school_chemistry": { "acc": 0.15270935960591134, "acc_stderr": 0.025308904539380624, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.8375286933057142, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_high_school_statistics": { "acc": 0.1527777777777778, "acc_stderr": 0.02453632602613422, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.7997694228225937, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_elementary_mathematics": { "acc": 0.20899470899470898, "acc_stderr": 0.020940481565334835, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.8093286036487656, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_electrical_engineering": { "acc": 0.2413793103448276, "acc_stderr": 0.03565998174135302, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.8067603588310134, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "mmlu_high_school_mathematics": { "acc": 0.2111111111111111, "acc_stderr": 0.02488211685765508, - "timestamp": "2024-11-12T00-51-50.090776" + "brier_score": 0.8071810033254354, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-02-55.059299" }, "arc_challenge": { "acc": 0.2158703071672355, diff --git a/metadata/model_scores/results_EleutherAI__pythia-6.9b.json b/metadata/model_scores/results_EleutherAI__pythia-6.9b.json index d137052..197ab89 100644 --- a/metadata/model_scores/results_EleutherAI__pythia-6.9b.json +++ b/metadata/model_scores/results_EleutherAI__pythia-6.9b.json @@ -1,6 +1,6 @@ { "model_name": "EleutherAI/pythia-6.9b", - "last_updated": "2024-12-04 11:24:23.872880", + "last_updated": "2024-12-19 13:40:15.926814", "results": { "harness": { "minerva_math_precalc": { @@ -228,289 +228,403 @@ "timestamp": "2024-06-07T21-53-35.445617" }, "mmlu_world_religions": { - "acc": 0.29239766081871343, - "acc_stderr": 0.034886477134579215, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.30994152046783624, + "acc_stderr": 0.03546976959393163, + "brier_score": 0.7607541360437958, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_formal_logic": { - "acc": 0.15873015873015872, - "acc_stderr": 0.03268454013011744, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.24603174603174602, + "acc_stderr": 0.03852273364924317, + "brier_score": 0.7558897175145493, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_prehistory": { - "acc": 0.23765432098765432, - "acc_stderr": 0.02368359183700856, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.2808641975308642, + "acc_stderr": 0.025006469755799215, + "brier_score": 0.7552343974599415, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_moral_scenarios": { "acc": 0.24692737430167597, - "acc_stderr": 0.014422292204808838, - "timestamp": "2024-11-12T22-04-18.010150" + "acc_stderr": 0.014422292204808842, + "brier_score": 0.7551737995162493, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_high_school_world_history": { - "acc": 0.2742616033755274, - "acc_stderr": 0.02904133351059804, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.24050632911392406, + "acc_stderr": 0.027820781981149678, + "brier_score": 0.76716563357048, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_moral_disputes": { - "acc": 0.2658959537572254, - "acc_stderr": 0.023786203255508283, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.27167630057803466, + "acc_stderr": 0.02394851290546835, + "brier_score": 0.7518613678618872, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_professional_law": { - "acc": 0.23859191655801826, - "acc_stderr": 0.010885929742002212, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.24119947848761408, + "acc_stderr": 0.010926496102034952, + "brier_score": 0.7695844539373945, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_logical_fallacies": { - "acc": 0.294478527607362, - "acc_stderr": 0.03581165790474082, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.3128834355828221, + "acc_stderr": 0.036429145782924055, + "brier_score": 0.7493157501646129, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_high_school_us_history": { - "acc": 0.2647058823529412, - "acc_stderr": 0.03096451792692339, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.27941176470588236, + "acc_stderr": 0.031493281045079556, + "brier_score": 0.7631207153722228, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_philosophy": { - "acc": 0.2733118971061093, - "acc_stderr": 0.025311765975426115, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.33440514469453375, + "acc_stderr": 0.02679542232789395, + "brier_score": 0.7436655510942353, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_jurisprudence": { - "acc": 0.3055555555555556, - "acc_stderr": 0.044531975073749834, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.26851851851851855, + "acc_stderr": 0.04284467968052192, + "brier_score": 0.7533622182470998, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_international_law": { - "acc": 0.38016528925619836, - "acc_stderr": 0.04431324501968432, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.35537190082644626, + "acc_stderr": 0.0436923632657398, + "brier_score": 0.7329535247802648, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_high_school_european_history": { - "acc": 0.24848484848484848, - "acc_stderr": 0.03374402644139405, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.21212121212121213, + "acc_stderr": 0.031922715695482974, + "brier_score": 0.7645629191281306, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_high_school_government_and_politics": { - "acc": 0.24352331606217617, - "acc_stderr": 0.03097543638684543, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.27979274611398963, + "acc_stderr": 0.032396370467357036, + "brier_score": 0.7541351413172132, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_high_school_microeconomics": { - "acc": 0.22268907563025211, - "acc_stderr": 0.02702543349888236, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.21428571428571427, + "acc_stderr": 0.026653531596715494, + "brier_score": 0.7603567565580419, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_high_school_geography": { - "acc": 0.2222222222222222, - "acc_stderr": 0.02962022787479048, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.25757575757575757, + "acc_stderr": 0.031156269519646847, + "brier_score": 0.7437409782398885, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_high_school_psychology": { - "acc": 0.23119266055045873, - "acc_stderr": 0.018075750241633163, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.24403669724770644, + "acc_stderr": 0.018415286351416406, + "brier_score": 0.7664984382654108, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_public_relations": { - "acc": 0.35454545454545455, - "acc_stderr": 0.045820048415054174, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.21818181818181817, + "acc_stderr": 0.03955932861795833, + "brier_score": 0.759405754385259, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_us_foreign_policy": { - "acc": 0.23, - "acc_stderr": 0.04229525846816508, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.22, + "acc_stderr": 0.04163331998932269, + "brier_score": 0.7606951982879532, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_sociology": { - "acc": 0.25870646766169153, - "acc_stderr": 0.030965903123573026, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.2537313432835821, + "acc_stderr": 0.030769444967296007, + "brier_score": 0.7634450081849491, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_high_school_macroeconomics": { - "acc": 0.30512820512820515, - "acc_stderr": 0.023346335293325887, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.24871794871794872, + "acc_stderr": 0.021916957709213796, + "brier_score": 0.7574546160387061, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_security_studies": { - "acc": 0.17551020408163265, - "acc_stderr": 0.024352800722970015, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.22040816326530613, + "acc_stderr": 0.026537045312145277, + "brier_score": 0.752254816581363, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_professional_psychology": { - "acc": 0.2679738562091503, - "acc_stderr": 0.017917974069594722, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.26633986928104575, + "acc_stderr": 0.017883188134667206, + "brier_score": 0.7565649200831607, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_human_sexuality": { "acc": 0.24427480916030533, "acc_stderr": 0.037683359597287434, - "timestamp": "2024-11-12T22-04-18.010150" + "brier_score": 0.7652780053590433, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_econometrics": { - "acc": 0.21929824561403508, - "acc_stderr": 0.03892431106518753, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.2631578947368421, + "acc_stderr": 0.0414243971948936, + "brier_score": 0.7652218900149195, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_miscellaneous": { - "acc": 0.29246487867177523, - "acc_stderr": 0.016267000684598645, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.26309067688378035, + "acc_stderr": 0.015745497169049046, + "brier_score": 0.7654908629478401, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_marketing": { - "acc": 0.2222222222222222, - "acc_stderr": 0.0272360139461967, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.28205128205128205, + "acc_stderr": 0.02948036054954119, + "brier_score": 0.7552332681529889, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_management": { - "acc": 0.27184466019417475, - "acc_stderr": 0.044052680241409216, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.2524271844660194, + "acc_stderr": 0.04301250399690878, + "brier_score": 0.7600245221684531, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_nutrition": { - "acc": 0.2581699346405229, - "acc_stderr": 0.025058503316958157, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.25163398692810457, + "acc_stderr": 0.024848018263875195, + "brier_score": 0.7640597340305756, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_medical_genetics": { - "acc": 0.24, - "acc_stderr": 0.04292346959909281, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.23, + "acc_stderr": 0.042295258468165065, + "brier_score": 0.7762216678033689, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_human_aging": { - "acc": 0.28699551569506726, - "acc_stderr": 0.030360379710291954, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.19282511210762332, + "acc_stderr": 0.026478240960489365, + "brier_score": 0.7800610696771287, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_professional_medicine": { - "acc": 0.3125, - "acc_stderr": 0.02815637344037142, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.22426470588235295, + "acc_stderr": 0.025336848563332362, + "brier_score": 0.7635379068383641, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_college_medicine": { - "acc": 0.19653179190751446, - "acc_stderr": 0.030299574664788147, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.23699421965317918, + "acc_stderr": 0.03242414757483098, + "brier_score": 0.7679840913432142, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_business_ethics": { - "acc": 0.25, - "acc_stderr": 0.04351941398892446, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.22, + "acc_stderr": 0.04163331998932269, + "brier_score": 0.7781903236182292, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_clinical_knowledge": { - "acc": 0.25660377358490566, - "acc_stderr": 0.02688064788905197, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.19622641509433963, + "acc_stderr": 0.02444238813110083, + "brier_score": 0.7681818346745, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_global_facts": { "acc": 0.33, - "acc_stderr": 0.04725815626252605, - "timestamp": "2024-11-12T22-04-18.010150" + "acc_stderr": 0.047258156262526045, + "brier_score": 0.7487369092626431, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_virology": { - "acc": 0.3192771084337349, - "acc_stderr": 0.036293353299478595, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.2469879518072289, + "acc_stderr": 0.03357351982064536, + "brier_score": 0.7695191849068174, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_professional_accounting": { - "acc": 0.29432624113475175, - "acc_stderr": 0.027187127011503796, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.2375886524822695, + "acc_stderr": 0.025389512552729906, + "brier_score": 0.7730920632947392, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_college_physics": { - "acc": 0.24509803921568626, - "acc_stderr": 0.04280105837364396, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.22549019607843138, + "acc_stderr": 0.041583075330832865, + "brier_score": 0.7921966035717108, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_high_school_physics": { - "acc": 0.25165562913907286, - "acc_stderr": 0.035433042343899844, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.2781456953642384, + "acc_stderr": 0.03658603262763743, + "brier_score": 0.7524004818521565, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_high_school_biology": { - "acc": 0.24838709677419354, - "acc_stderr": 0.024580028921480992, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.23870967741935484, + "acc_stderr": 0.024251071262208834, + "brier_score": 0.7597133258999512, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_college_biology": { - "acc": 0.2777777777777778, - "acc_stderr": 0.03745554791462457, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.2638888888888889, + "acc_stderr": 0.03685651095897532, + "brier_score": 0.7732549139379925, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_anatomy": { - "acc": 0.34814814814814815, - "acc_stderr": 0.041153246103369526, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.34074074074074073, + "acc_stderr": 0.040943762699967946, + "brier_score": 0.7401866268979679, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_college_chemistry": { "acc": 0.18, - "acc_stderr": 0.03861229196653696, - "timestamp": "2024-11-12T22-04-18.010150" + "acc_stderr": 0.038612291966536955, + "brier_score": 0.7873479975290243, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_computer_security": { - "acc": 0.28, - "acc_stderr": 0.04512608598542127, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 0.7573995889302786, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_college_computer_science": { - "acc": 0.3, - "acc_stderr": 0.046056618647183814, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.27, + "acc_stderr": 0.04461960433384741, + "brier_score": 0.7698551965779535, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_astronomy": { - "acc": 0.29605263157894735, - "acc_stderr": 0.03715062154998905, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.3223684210526316, + "acc_stderr": 0.038035102483515854, + "brier_score": 0.732576931227996, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_college_mathematics": { - "acc": 0.22, - "acc_stderr": 0.04163331998932269, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.26, + "acc_stderr": 0.044084400227680794, + "brier_score": 0.7955437721978141, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_conceptual_physics": { - "acc": 0.32340425531914896, - "acc_stderr": 0.030579442773610348, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.20425531914893616, + "acc_stderr": 0.026355158413349414, + "brier_score": 0.7729307589058161, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_abstract_algebra": { - "acc": 0.25, - "acc_stderr": 0.04351941398892446, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "brier_score": 0.7534459394903233, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_high_school_computer_science": { - "acc": 0.33, - "acc_stderr": 0.04725815626252605, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "brier_score": 0.7555446493228442, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_machine_learning": { - "acc": 0.26785714285714285, - "acc_stderr": 0.04203277291467762, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.23214285714285715, + "acc_stderr": 0.04007341809755805, + "brier_score": 0.7669895465473596, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_high_school_chemistry": { - "acc": 0.2512315270935961, - "acc_stderr": 0.030516530732694436, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.30049261083743845, + "acc_stderr": 0.03225799476233486, + "brier_score": 0.7544466344426068, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_high_school_statistics": { - "acc": 0.3425925925925926, - "acc_stderr": 0.03236585252602157, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.2962962962962963, + "acc_stderr": 0.03114144782353604, + "brier_score": 0.7459779406514874, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_elementary_mathematics": { - "acc": 0.2830687830687831, - "acc_stderr": 0.023201392938194978, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.2619047619047619, + "acc_stderr": 0.022644212615525218, + "brier_score": 0.7902749238124076, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_electrical_engineering": { - "acc": 0.27586206896551724, - "acc_stderr": 0.037245636197746325, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.31724137931034485, + "acc_stderr": 0.038783523721386215, + "brier_score": 0.7439372516923184, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "mmlu_high_school_mathematics": { - "acc": 0.2111111111111111, - "acc_stderr": 0.02488211685765508, - "timestamp": "2024-11-12T22-04-18.010150" + "acc": 0.26296296296296295, + "acc_stderr": 0.026842057873833706, + "brier_score": 0.8017153999088171, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-13-55.298554" }, "arc_challenge": { "acc": 0.3575085324232082, diff --git a/metadata/model_scores/results_EleutherAI__pythia-70m-deduped.json b/metadata/model_scores/results_EleutherAI__pythia-70m-deduped.json index c34bd43..aa515cc 100644 --- a/metadata/model_scores/results_EleutherAI__pythia-70m-deduped.json +++ b/metadata/model_scores/results_EleutherAI__pythia-70m-deduped.json @@ -1,6 +1,6 @@ { "model_name": "EleutherAI/pythia-70m-deduped", - "last_updated": "2024-12-04 11:25:28.338970", + "last_updated": "2024-12-19 13:41:39.605279", "results": { "harness": { "drop": { @@ -561,289 +561,403 @@ } }, "mmlu_world_religions": { - "acc": 0.3216374269005848, - "acc_stderr": 0.035825294425731215, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.3157894736842105, + "acc_stderr": 0.03565079670708313, + "brier_score": 0.9458017124200467, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_formal_logic": { - "acc": 0.21428571428571427, - "acc_stderr": 0.03670066451047182, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.2698412698412698, + "acc_stderr": 0.03970158273235172, + "brier_score": 0.9944529998737204, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_prehistory": { - "acc": 0.25308641975308643, - "acc_stderr": 0.024191808600713, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.21296296296296297, + "acc_stderr": 0.022779719088733396, + "brier_score": 1.1104097102004094, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_moral_scenarios": { - "acc": 0.2581005586592179, - "acc_stderr": 0.014635185616527832, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.23798882681564246, + "acc_stderr": 0.014242630070574885, + "brier_score": 1.0107834268490483, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_high_school_world_history": { - "acc": 0.29535864978902954, - "acc_stderr": 0.02969633871342289, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.270042194092827, + "acc_stderr": 0.028900721906293426, + "brier_score": 1.0541749946967076, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_moral_disputes": { - "acc": 0.26011560693641617, - "acc_stderr": 0.023618678310069363, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.24855491329479767, + "acc_stderr": 0.023267528432100174, + "brier_score": 1.1503169332598566, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_professional_law": { - "acc": 0.23663624511082137, - "acc_stderr": 0.010855137351572739, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.2470664928292047, + "acc_stderr": 0.011015752255279329, + "brier_score": 1.0777841379995983, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_logical_fallacies": { - "acc": 0.27607361963190186, - "acc_stderr": 0.0351238528370505, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.22085889570552147, + "acc_stderr": 0.032591773927421776, + "brier_score": 1.1873647876473623, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_high_school_us_history": { - "acc": 0.29411764705882354, - "acc_stderr": 0.03198001660115072, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.25, + "acc_stderr": 0.03039153369274154, + "brier_score": 1.0797949488812022, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_philosophy": { - "acc": 0.19292604501607716, - "acc_stderr": 0.022411516780911363, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.1864951768488746, + "acc_stderr": 0.02212243977248077, + "brier_score": 1.285858430936262, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_jurisprudence": { - "acc": 0.21296296296296297, - "acc_stderr": 0.0395783547198098, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.25925925925925924, + "acc_stderr": 0.04236511258094634, + "brier_score": 1.113612046428483, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_international_law": { - "acc": 0.256198347107438, - "acc_stderr": 0.03984979653302871, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.2396694214876033, + "acc_stderr": 0.03896878985070417, + "brier_score": 1.1609394278395657, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_high_school_european_history": { - "acc": 0.2606060606060606, - "acc_stderr": 0.034277431758165236, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.21818181818181817, + "acc_stderr": 0.03225078108306289, + "brier_score": 1.0797849216735733, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_high_school_government_and_politics": { - "acc": 0.3626943005181347, - "acc_stderr": 0.034697137917043715, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.19689119170984457, + "acc_stderr": 0.02869787397186069, + "brier_score": 1.1874656538908335, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_high_school_microeconomics": { - "acc": 0.22268907563025211, - "acc_stderr": 0.027025433498882367, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.21008403361344538, + "acc_stderr": 0.026461398717471874, + "brier_score": 1.1303851879741371, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_high_school_geography": { - "acc": 0.20202020202020202, - "acc_stderr": 0.028606204289229865, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.17676767676767677, + "acc_stderr": 0.027178752639044915, + "brier_score": 1.145581139934715, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_high_school_psychology": { - "acc": 0.23486238532110093, - "acc_stderr": 0.01817511051034359, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.1944954128440367, + "acc_stderr": 0.01697028909045803, + "brier_score": 1.172174012955203, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_public_relations": { - "acc": 0.20909090909090908, - "acc_stderr": 0.03895091015724137, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.21818181818181817, + "acc_stderr": 0.03955932861795833, + "brier_score": 1.1841312871839917, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_us_foreign_policy": { - "acc": 0.31, - "acc_stderr": 0.04648231987117316, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.28, + "acc_stderr": 0.045126085985421276, + "brier_score": 1.0080093333768496, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_sociology": { - "acc": 0.24875621890547264, - "acc_stderr": 0.030567675938916718, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.24378109452736318, + "acc_stderr": 0.030360490154014652, + "brier_score": 1.1267198627231036, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_high_school_macroeconomics": { - "acc": 0.2948717948717949, - "acc_stderr": 0.023119362758232277, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.20256410256410257, + "acc_stderr": 0.020377660970371397, + "brier_score": 1.1203296493874824, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_security_studies": { - "acc": 0.2163265306122449, - "acc_stderr": 0.02635891633490401, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.18775510204081633, + "acc_stderr": 0.02500025603954622, + "brier_score": 1.2394159309129527, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_professional_psychology": { - "acc": 0.24836601307189543, - "acc_stderr": 0.017479487001364764, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.25, + "acc_stderr": 0.01751781884501444, + "brier_score": 1.0924564010576683, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_human_sexuality": { - "acc": 0.2900763358778626, - "acc_stderr": 0.03980066246467765, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.2595419847328244, + "acc_stderr": 0.03844876139785271, + "brier_score": 1.007720693073942, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_econometrics": { - "acc": 0.2719298245614035, - "acc_stderr": 0.04185774424022056, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.23684210526315788, + "acc_stderr": 0.039994238792813386, + "brier_score": 1.0545954545627128, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_miscellaneous": { - "acc": 0.24393358876117496, - "acc_stderr": 0.015357212665829489, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.23754789272030652, + "acc_stderr": 0.015218733046150195, + "brier_score": 1.0303813244930453, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_marketing": { - "acc": 0.20085470085470086, - "acc_stderr": 0.02624677294689048, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.2905982905982906, + "acc_stderr": 0.029745048572674057, + "brier_score": 1.0758487519425721, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_management": { - "acc": 0.1941747572815534, - "acc_stderr": 0.03916667762822584, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.17475728155339806, + "acc_stderr": 0.03760178006026621, + "brier_score": 1.1930902498012064, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_nutrition": { - "acc": 0.2875816993464052, - "acc_stderr": 0.02591780611714716, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.2222222222222222, + "acc_stderr": 0.023805186524888142, + "brier_score": 1.0089845272533258, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_medical_genetics": { - "acc": 0.33, - "acc_stderr": 0.047258156262526045, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "brier_score": 0.9776851574403502, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_human_aging": { - "acc": 0.2600896860986547, - "acc_stderr": 0.029442495585857487, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.31390134529147984, + "acc_stderr": 0.03114679648297246, + "brier_score": 0.9962092883560694, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_professional_medicine": { - "acc": 0.41911764705882354, - "acc_stderr": 0.02997280717046463, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.1875, + "acc_stderr": 0.023709788253811766, + "brier_score": 1.0150739563833298, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_college_medicine": { - "acc": 0.19653179190751446, - "acc_stderr": 0.030299574664788137, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.2138728323699422, + "acc_stderr": 0.03126511206173043, + "brier_score": 1.0742749717473168, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_business_ethics": { - "acc": 0.2, - "acc_stderr": 0.04020151261036845, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "brier_score": 0.9929656337361632, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_clinical_knowledge": { - "acc": 0.2528301886792453, - "acc_stderr": 0.026749899771241235, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.21132075471698114, + "acc_stderr": 0.025125766484827856, + "brier_score": 1.0657689854635248, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_global_facts": { - "acc": 0.17, - "acc_stderr": 0.0377525168068637, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.18, + "acc_stderr": 0.038612291966536955, + "brier_score": 1.053537895714484, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_virology": { - "acc": 0.1746987951807229, - "acc_stderr": 0.029560326211256833, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.28313253012048195, + "acc_stderr": 0.03507295431370518, + "brier_score": 1.0126987671317704, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_professional_accounting": { - "acc": 0.24468085106382978, - "acc_stderr": 0.025645553622266733, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.23404255319148937, + "acc_stderr": 0.025257861359432407, + "brier_score": 1.0677264210796846, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_college_physics": { - "acc": 0.23529411764705882, - "acc_stderr": 0.04220773659171452, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.21568627450980393, + "acc_stderr": 0.040925639582376556, + "brier_score": 1.0123830470834905, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_high_school_physics": { - "acc": 0.17880794701986755, - "acc_stderr": 0.03128744850600724, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.19205298013245034, + "acc_stderr": 0.032162984205936135, + "brier_score": 1.0638867246138666, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_high_school_biology": { - "acc": 0.3161290322580645, - "acc_stderr": 0.02645087448904277, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.1774193548387097, + "acc_stderr": 0.021732540689329265, + "brier_score": 1.1388412832167263, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_college_biology": { - "acc": 0.2152777777777778, - "acc_stderr": 0.034370793441061344, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.2638888888888889, + "acc_stderr": 0.03685651095897532, + "brier_score": 1.0202379818652771, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_anatomy": { - "acc": 0.32592592592592595, - "acc_stderr": 0.040491220417025055, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.1925925925925926, + "acc_stderr": 0.03406542058502653, + "brier_score": 1.0443440972173736, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_college_chemistry": { - "acc": 0.29, - "acc_stderr": 0.045604802157206845, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.19, + "acc_stderr": 0.03942772444036623, + "brier_score": 0.9933008024295017, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_computer_security": { - "acc": 0.26, - "acc_stderr": 0.0440844002276808, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.29, + "acc_stderr": 0.045604802157206845, + "brier_score": 0.9582996285251297, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_college_computer_science": { - "acc": 0.32, - "acc_stderr": 0.046882617226215034, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 1.018394924584262, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_astronomy": { - "acc": 0.19078947368421054, - "acc_stderr": 0.031975658210325, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.17763157894736842, + "acc_stderr": 0.031103182383123398, + "brier_score": 1.1303538216548912, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_college_mathematics": { - "acc": 0.25, - "acc_stderr": 0.04351941398892446, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "brier_score": 1.0666907621711257, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_conceptual_physics": { - "acc": 0.28936170212765955, - "acc_stderr": 0.02964400657700962, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.26382978723404255, + "acc_stderr": 0.02880998985410298, + "brier_score": 1.0360568049709666, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_abstract_algebra": { - "acc": 0.26, - "acc_stderr": 0.0440844002276808, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.22, + "acc_stderr": 0.04163331998932269, + "brier_score": 1.15021593285239, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_high_school_computer_science": { "acc": 0.26, - "acc_stderr": 0.04408440022768078, - "timestamp": "2024-11-20T02-41-14.007364" + "acc_stderr": 0.04408440022768079, + "brier_score": 0.9914560507266944, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_machine_learning": { - "acc": 0.22321428571428573, - "acc_stderr": 0.039523019677025116, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.33035714285714285, + "acc_stderr": 0.04464285714285713, + "brier_score": 0.9066774590600205, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_high_school_chemistry": { - "acc": 0.27586206896551724, - "acc_stderr": 0.03144712581678241, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.16748768472906403, + "acc_stderr": 0.02627308604753542, + "brier_score": 1.011348400920999, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_high_school_statistics": { - "acc": 0.46296296296296297, - "acc_stderr": 0.03400603625538271, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.1527777777777778, + "acc_stderr": 0.02453632602613422, + "brier_score": 1.1822905019744807, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_elementary_mathematics": { - "acc": 0.2566137566137566, - "acc_stderr": 0.022494510767503154, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.20899470899470898, + "acc_stderr": 0.020940481565334835, + "brier_score": 1.0424135555350613, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_electrical_engineering": { - "acc": 0.20689655172413793, - "acc_stderr": 0.03375672449560553, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.2413793103448276, + "acc_stderr": 0.03565998174135302, + "brier_score": 1.0297063015300025, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "mmlu_high_school_mathematics": { - "acc": 0.22592592592592592, - "acc_stderr": 0.025497532639609553, - "timestamp": "2024-11-20T02-41-14.007364" + "acc": 0.2111111111111111, + "acc_stderr": 0.02488211685765508, + "brier_score": 0.9946655374318092, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-22-21.895675" }, "arc_challenge": { "acc": 0.1697952218430034, diff --git a/metadata/model_scores/results_HuggingFaceTB__SmolLM-1.7B.json b/metadata/model_scores/results_HuggingFaceTB__SmolLM-1.7B.json index b3d5910..4be4bea 100644 --- a/metadata/model_scores/results_HuggingFaceTB__SmolLM-1.7B.json +++ b/metadata/model_scores/results_HuggingFaceTB__SmolLM-1.7B.json @@ -1,292 +1,406 @@ { "model_name": "HuggingFaceTB__SmolLM-1.7B", - "last_updated": "2024-12-04 11:25:37.408937", + "last_updated": "2024-12-19 13:41:49.711330", "results": { "harness": { "mmlu_world_religions": { - "acc": 0.34502923976608185, - "acc_stderr": 0.03645981377388807, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.30409356725146197, + "acc_stderr": 0.03528211258245232, + "brier_score": 0.7551636436309582, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_formal_logic": { - "acc": 0.2222222222222222, - "acc_stderr": 0.037184890068181146, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.21428571428571427, + "acc_stderr": 0.03670066451047182, + "brier_score": 0.7801281626265447, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_prehistory": { - "acc": 0.3271604938271605, - "acc_stderr": 0.026105673861409814, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.3117283950617284, + "acc_stderr": 0.025773111169630433, + "brier_score": 0.7327440907985833, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_moral_scenarios": { - "acc": 0.2446927374301676, - "acc_stderr": 0.014378169884098424, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.24692737430167597, + "acc_stderr": 0.014422292204808838, + "brier_score": 0.7665170648069967, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_high_school_world_history": { - "acc": 0.26582278481012656, - "acc_stderr": 0.028756799629658332, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.32489451476793246, + "acc_stderr": 0.030486039389105296, + "brier_score": 0.7430802455771087, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_moral_disputes": { - "acc": 0.3236994219653179, - "acc_stderr": 0.02519018132760842, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.32947976878612717, + "acc_stderr": 0.025305258131879723, + "brier_score": 0.7487056188535807, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_professional_law": { - "acc": 0.2835723598435463, - "acc_stderr": 0.011511900775968318, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.2666232073011734, + "acc_stderr": 0.01129383603161214, + "brier_score": 0.7597029785420658, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_logical_fallacies": { - "acc": 0.26380368098159507, - "acc_stderr": 0.03462419931615624, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.25766871165644173, + "acc_stderr": 0.03436150827846917, + "brier_score": 0.7527711239704512, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_high_school_us_history": { - "acc": 0.23039215686274508, - "acc_stderr": 0.029554292605695063, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.30392156862745096, + "acc_stderr": 0.03228210387037892, + "brier_score": 0.7470402138960359, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_philosophy": { - "acc": 0.36977491961414793, - "acc_stderr": 0.027417996705630995, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.33440514469453375, + "acc_stderr": 0.026795422327893937, + "brier_score": 0.7318698523628182, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_jurisprudence": { - "acc": 0.3055555555555556, - "acc_stderr": 0.044531975073749834, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.32407407407407407, + "acc_stderr": 0.04524596007030048, + "brier_score": 0.7467307283787081, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_international_law": { - "acc": 0.36363636363636365, - "acc_stderr": 0.043913262867240704, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.4214876033057851, + "acc_stderr": 0.045077322787750944, + "brier_score": 0.7196812416252452, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_high_school_european_history": { - "acc": 0.24242424242424243, - "acc_stderr": 0.033464098810559534, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.30303030303030304, + "acc_stderr": 0.035886248000917095, + "brier_score": 0.7466180394615518, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_high_school_government_and_politics": { - "acc": 0.27461139896373055, - "acc_stderr": 0.032210245080411544, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.2538860103626943, + "acc_stderr": 0.03141024780565319, + "brier_score": 0.7520148085554117, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_high_school_microeconomics": { - "acc": 0.2605042016806723, - "acc_stderr": 0.02851025151234193, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.23529411764705882, + "acc_stderr": 0.02755361446786381, + "brier_score": 0.7706544205273947, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_high_school_geography": { - "acc": 0.3484848484848485, - "acc_stderr": 0.033948539651564025, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.2676767676767677, + "acc_stderr": 0.03154449888270286, + "brier_score": 0.7539202035996062, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_high_school_psychology": { - "acc": 0.30458715596330277, - "acc_stderr": 0.019732299420354045, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.25688073394495414, + "acc_stderr": 0.018732492928342444, + "brier_score": 0.7540537370882963, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_public_relations": { - "acc": 0.2545454545454545, - "acc_stderr": 0.04172343038705383, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.3090909090909091, + "acc_stderr": 0.044262946482000985, + "brier_score": 0.741642452100577, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_us_foreign_policy": { - "acc": 0.39, - "acc_stderr": 0.04902071300001974, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "brier_score": 0.7395676551979159, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_sociology": { "acc": 0.2885572139303483, - "acc_stderr": 0.0320384104021332, - "timestamp": "2024-11-21T21-51-42.336660" + "acc_stderr": 0.03203841040213321, + "brier_score": 0.7494414822803905, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_high_school_macroeconomics": { - "acc": 0.28974358974358977, - "acc_stderr": 0.023000628243687954, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.26153846153846155, + "acc_stderr": 0.022282141204204423, + "brier_score": 0.7690134510038235, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_security_studies": { - "acc": 0.363265306122449, - "acc_stderr": 0.030789051139030806, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.24897959183673468, + "acc_stderr": 0.027682979522960227, + "brier_score": 0.760649006945786, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_professional_psychology": { - "acc": 0.26633986928104575, - "acc_stderr": 0.01788318813466721, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.27124183006535946, + "acc_stderr": 0.01798661530403032, + "brier_score": 0.7559105031184763, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_human_sexuality": { "acc": 0.29770992366412213, - "acc_stderr": 0.04010358942462203, - "timestamp": "2024-11-21T21-51-42.336660" + "acc_stderr": 0.040103589424622034, + "brier_score": 0.740708587662364, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_econometrics": { - "acc": 0.20175438596491227, - "acc_stderr": 0.03775205013583639, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.21052631578947367, + "acc_stderr": 0.038351539543994194, + "brier_score": 0.7576774223011613, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_miscellaneous": { - "acc": 0.33205619412515963, - "acc_stderr": 0.016841174655295717, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.31417624521072796, + "acc_stderr": 0.0165992917358849, + "brier_score": 0.7455891824240866, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_marketing": { - "acc": 0.2863247863247863, - "acc_stderr": 0.029614323690456648, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.3162393162393162, + "acc_stderr": 0.030463656747340244, + "brier_score": 0.7519586934939722, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_management": { - "acc": 0.30097087378640774, - "acc_stderr": 0.04541609446503948, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.2815533980582524, + "acc_stderr": 0.04453254836326469, + "brier_score": 0.759008869776314, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_nutrition": { - "acc": 0.3333333333333333, - "acc_stderr": 0.02699254433929723, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.29411764705882354, + "acc_stderr": 0.02609016250427905, + "brier_score": 0.7566521828898929, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_medical_genetics": { - "acc": 0.32, - "acc_stderr": 0.04688261722621504, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.28, + "acc_stderr": 0.04512608598542128, + "brier_score": 0.7584183884384231, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_human_aging": { - "acc": 0.33183856502242154, - "acc_stderr": 0.031602951437766785, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.28699551569506726, + "acc_stderr": 0.03036037971029196, + "brier_score": 0.7601582506539263, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_professional_medicine": { - "acc": 0.3492647058823529, - "acc_stderr": 0.02895975519682487, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.2536764705882353, + "acc_stderr": 0.02643132987078954, + "brier_score": 0.7525283294492598, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_college_medicine": { - "acc": 0.2658959537572254, - "acc_stderr": 0.0336876293225943, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.23699421965317918, + "acc_stderr": 0.03242414757483098, + "brier_score": 0.7557866419534296, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_business_ethics": { - "acc": 0.32, - "acc_stderr": 0.046882617226215034, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.28, + "acc_stderr": 0.04512608598542128, + "brier_score": 0.7593669275619496, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_clinical_knowledge": { - "acc": 0.35094339622641507, - "acc_stderr": 0.029373646253234686, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.2641509433962264, + "acc_stderr": 0.02713429162874172, + "brier_score": 0.7559920747609482, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_global_facts": { - "acc": 0.35, - "acc_stderr": 0.0479372485441102, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.36, + "acc_stderr": 0.04824181513244218, + "brier_score": 0.7280857545855145, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_virology": { - "acc": 0.3795180722891566, - "acc_stderr": 0.03777798822748018, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.2891566265060241, + "acc_stderr": 0.03529486801511115, + "brier_score": 0.7417908992843658, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_professional_accounting": { - "acc": 0.24113475177304963, - "acc_stderr": 0.02551873104953776, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.2553191489361702, + "acc_stderr": 0.026011992930902006, + "brier_score": 0.7609243491199253, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_college_physics": { - "acc": 0.2647058823529412, - "acc_stderr": 0.04389869956808779, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.19607843137254902, + "acc_stderr": 0.03950581861179963, + "brier_score": 0.7712757539051257, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_high_school_physics": { - "acc": 0.271523178807947, - "acc_stderr": 0.03631329803969653, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.2847682119205298, + "acc_stderr": 0.03684881521389024, + "brier_score": 0.7527327446093819, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_high_school_biology": { - "acc": 0.33225806451612905, - "acc_stderr": 0.026795560848122797, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.2903225806451613, + "acc_stderr": 0.025822106119415898, + "brier_score": 0.7558582574449789, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_college_biology": { - "acc": 0.3541666666666667, - "acc_stderr": 0.039994111357535424, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.2569444444444444, + "acc_stderr": 0.03653946969442099, + "brier_score": 0.7742859530237306, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_anatomy": { - "acc": 0.4148148148148148, - "acc_stderr": 0.04256193767901407, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.32592592592592595, + "acc_stderr": 0.040491220417025055, + "brier_score": 0.7313849868720731, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_college_chemistry": { - "acc": 0.32, - "acc_stderr": 0.046882617226215034, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 0.7727946797892771, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_computer_security": { - "acc": 0.31, - "acc_stderr": 0.04648231987117316, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.27, + "acc_stderr": 0.0446196043338474, + "brier_score": 0.740939384922402, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_college_computer_science": { - "acc": 0.34, - "acc_stderr": 0.04760952285695235, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.2, + "acc_stderr": 0.04020151261036846, + "brier_score": 0.7795700638432723, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_astronomy": { - "acc": 0.3618421052631579, - "acc_stderr": 0.03910525752849725, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.3223684210526316, + "acc_stderr": 0.03803510248351586, + "brier_score": 0.7392204275275199, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_college_mathematics": { - "acc": 0.27, - "acc_stderr": 0.044619604333847394, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 0.76147059626292, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_conceptual_physics": { - "acc": 0.28085106382978725, - "acc_stderr": 0.02937917046412482, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.2936170212765957, + "acc_stderr": 0.029771642712491227, + "brier_score": 0.7448651469298134, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_abstract_algebra": { - "acc": 0.24, - "acc_stderr": 0.04292346959909283, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.27, + "acc_stderr": 0.0446196043338474, + "brier_score": 0.7693922391618123, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_high_school_computer_science": { - "acc": 0.33, - "acc_stderr": 0.047258156262526045, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.32, + "acc_stderr": 0.04688261722621504, + "brier_score": 0.7305897004268131, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_machine_learning": { - "acc": 0.32142857142857145, - "acc_stderr": 0.04432804055291518, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.25, + "acc_stderr": 0.04109974682633932, + "brier_score": 0.7620884806930387, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_high_school_chemistry": { - "acc": 0.2413793103448276, - "acc_stderr": 0.030108330718011625, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.2660098522167488, + "acc_stderr": 0.03108982600293752, + "brier_score": 0.7493879817389563, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_high_school_statistics": { - "acc": 0.3194444444444444, - "acc_stderr": 0.0317987634217685, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.17592592592592593, + "acc_stderr": 0.025967420958258526, + "brier_score": 0.7940560528283264, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_elementary_mathematics": { - "acc": 0.24867724867724866, - "acc_stderr": 0.022261817692400182, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.2724867724867725, + "acc_stderr": 0.02293097307163334, + "brier_score": 0.7668511170348982, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_electrical_engineering": { - "acc": 0.2206896551724138, - "acc_stderr": 0.0345593020192481, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.2689655172413793, + "acc_stderr": 0.036951833116502325, + "brier_score": 0.7554425454586065, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "mmlu_high_school_mathematics": { - "acc": 0.24814814814814815, - "acc_stderr": 0.0263357394040558, - "timestamp": "2024-11-21T21-51-42.336660" + "acc": 0.26296296296296295, + "acc_stderr": 0.026842057873833706, + "brier_score": 0.7690530697397449, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-02-17.509556" }, "arc_challenge": { "acc": 0.4616040955631399, diff --git a/metadata/model_scores/results_HuggingFaceTB__SmolLM-360M.json b/metadata/model_scores/results_HuggingFaceTB__SmolLM-360M.json index db9c973..25df23b 100644 --- a/metadata/model_scores/results_HuggingFaceTB__SmolLM-360M.json +++ b/metadata/model_scores/results_HuggingFaceTB__SmolLM-360M.json @@ -1,292 +1,406 @@ { "model_name": "HuggingFaceTB__SmolLM-360M", - "last_updated": "2024-12-04 11:24:56.356998", + "last_updated": "2024-12-19 13:40:57.063141", "results": { "harness": { "mmlu_world_religions": { - "acc": 0.28654970760233917, - "acc_stderr": 0.03467826685703826, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.3157894736842105, + "acc_stderr": 0.03565079670708312, + "brier_score": 0.7489952381826336, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_formal_logic": { - "acc": 0.1746031746031746, - "acc_stderr": 0.03395490020856113, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.3253968253968254, + "acc_stderr": 0.04190596438871137, + "brier_score": 0.744205156755892, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_prehistory": { - "acc": 0.2808641975308642, - "acc_stderr": 0.025006469755799208, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.22530864197530864, + "acc_stderr": 0.023246202647819753, + "brier_score": 0.7558066141669159, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_moral_scenarios": { - "acc": 0.24692737430167597, - "acc_stderr": 0.014422292204808838, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.2424581005586592, + "acc_stderr": 0.014333522059217892, + "brier_score": 0.7580088742032026, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_high_school_world_history": { - "acc": 0.27848101265822783, - "acc_stderr": 0.029178682304842555, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.3080168776371308, + "acc_stderr": 0.0300523893356057, + "brier_score": 0.7481386433912905, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_moral_disputes": { - "acc": 0.2832369942196532, - "acc_stderr": 0.024257901705323374, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.26011560693641617, + "acc_stderr": 0.023618678310069363, + "brier_score": 0.7537079980328932, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_professional_law": { - "acc": 0.23989569752281617, - "acc_stderr": 0.010906282617981636, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.24185136897001303, + "acc_stderr": 0.010936550813827075, + "brier_score": 0.7553302357472506, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_logical_fallacies": { - "acc": 0.294478527607362, - "acc_stderr": 0.03581165790474082, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.2147239263803681, + "acc_stderr": 0.032262193772867744, + "brier_score": 0.7586846766296248, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_high_school_us_history": { - "acc": 0.19117647058823528, - "acc_stderr": 0.027599174300640773, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.22549019607843138, + "acc_stderr": 0.02933116229425173, + "brier_score": 0.7579910312618601, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_philosophy": { - "acc": 0.2861736334405145, - "acc_stderr": 0.025670259242188936, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.20257234726688103, + "acc_stderr": 0.022827317491059675, + "brier_score": 0.7625719281567497, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_jurisprudence": { - "acc": 0.18518518518518517, - "acc_stderr": 0.03755265865037183, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.2222222222222222, + "acc_stderr": 0.0401910747255735, + "brier_score": 0.758042935757606, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_international_law": { - "acc": 0.38016528925619836, - "acc_stderr": 0.04431324501968432, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.2727272727272727, + "acc_stderr": 0.04065578140908705, + "brier_score": 0.7402562726111097, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_high_school_european_history": { - "acc": 0.24242424242424243, - "acc_stderr": 0.03346409881055953, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.28484848484848485, + "acc_stderr": 0.035243908445117836, + "brier_score": 0.7518476305982694, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_high_school_government_and_politics": { - "acc": 0.25906735751295334, - "acc_stderr": 0.0316187791793541, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.21761658031088082, + "acc_stderr": 0.02977866303775297, + "brier_score": 0.7561294622120225, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_high_school_microeconomics": { "acc": 0.23529411764705882, - "acc_stderr": 0.02755361446786381, - "timestamp": "2024-11-10T15-21-10.938137" + "acc_stderr": 0.027553614467863818, + "brier_score": 0.7507375033633016, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_high_school_geography": { - "acc": 0.23737373737373738, - "acc_stderr": 0.030313710538198896, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.23232323232323232, + "acc_stderr": 0.030088629490217483, + "brier_score": 0.7523217766742214, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_high_school_psychology": { - "acc": 0.23669724770642203, - "acc_stderr": 0.018224078117299078, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.2018348623853211, + "acc_stderr": 0.01720857935778759, + "brier_score": 0.7599936909886118, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_public_relations": { - "acc": 0.2727272727272727, - "acc_stderr": 0.04265792110940588, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.22727272727272727, + "acc_stderr": 0.04013964554072775, + "brier_score": 0.7501591730485774, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_us_foreign_policy": { - "acc": 0.24, - "acc_stderr": 0.04292346959909282, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.29, + "acc_stderr": 0.04560480215720684, + "brier_score": 0.7527221718432113, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_sociology": { - "acc": 0.2537313432835821, - "acc_stderr": 0.030769444967296007, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.21890547263681592, + "acc_stderr": 0.029239174636647, + "brier_score": 0.7640614797494962, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_high_school_macroeconomics": { - "acc": 0.2282051282051282, - "acc_stderr": 0.02127839386358628, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.24358974358974358, + "acc_stderr": 0.02176373368417391, + "brier_score": 0.7528827648024928, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_security_studies": { "acc": 0.20408163265306123, - "acc_stderr": 0.0258012834750905, - "timestamp": "2024-11-10T15-21-10.938137" + "acc_stderr": 0.025801283475090513, + "brier_score": 0.7677579726501517, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_professional_psychology": { - "acc": 0.25163398692810457, - "acc_stderr": 0.017555818091322284, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.25326797385620914, + "acc_stderr": 0.017593486895366835, + "brier_score": 0.7584410341064913, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_human_sexuality": { - "acc": 0.20610687022900764, - "acc_stderr": 0.035477710041594626, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.25190839694656486, + "acc_stderr": 0.038073871163060866, + "brier_score": 0.7628773720030533, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_econometrics": { - "acc": 0.2631578947368421, - "acc_stderr": 0.041424397194893624, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.2719298245614035, + "acc_stderr": 0.041857744240220575, + "brier_score": 0.7629268418474607, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_miscellaneous": { - "acc": 0.2784163473818646, - "acc_stderr": 0.01602829518899246, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.27586206896551724, + "acc_stderr": 0.015982814774695625, + "brier_score": 0.7496129073986078, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_marketing": { - "acc": 0.2222222222222222, - "acc_stderr": 0.027236013946196704, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.27350427350427353, + "acc_stderr": 0.029202540153431173, + "brier_score": 0.7543714738187707, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_management": { - "acc": 0.1553398058252427, - "acc_stderr": 0.03586594738573973, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.2815533980582524, + "acc_stderr": 0.04453254836326467, + "brier_score": 0.7475248395770401, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_nutrition": { - "acc": 0.22875816993464052, - "acc_stderr": 0.02405102973991225, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.23529411764705882, + "acc_stderr": 0.024288619466046116, + "brier_score": 0.7556467124494557, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_medical_genetics": { - "acc": 0.27, - "acc_stderr": 0.044619604333847394, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "brier_score": 0.7351355841978402, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_human_aging": { - "acc": 0.32286995515695066, - "acc_stderr": 0.03138147637575498, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.336322869955157, + "acc_stderr": 0.031708824268455, + "brier_score": 0.7448421549573002, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_professional_medicine": { - "acc": 0.39338235294117646, - "acc_stderr": 0.02967428828131118, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.2977941176470588, + "acc_stderr": 0.027778298701545443, + "brier_score": 0.7454242717298569, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_college_medicine": { - "acc": 0.19653179190751446, - "acc_stderr": 0.030299574664788147, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.20809248554913296, + "acc_stderr": 0.030952890217749867, + "brier_score": 0.7532253481406085, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_business_ethics": { - "acc": 0.24, - "acc_stderr": 0.04292346959909283, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.26, + "acc_stderr": 0.0440844002276808, + "brier_score": 0.7491100760740861, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_clinical_knowledge": { - "acc": 0.2641509433962264, - "acc_stderr": 0.027134291628741713, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.25660377358490566, + "acc_stderr": 0.026880647889051996, + "brier_score": 0.7507402312612156, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_global_facts": { - "acc": 0.32, - "acc_stderr": 0.04688261722621505, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 0.7467218803779668, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_virology": { - "acc": 0.30120481927710846, - "acc_stderr": 0.03571609230053481, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.2891566265060241, + "acc_stderr": 0.03529486801511115, + "brier_score": 0.7489050763698124, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_professional_accounting": { - "acc": 0.25886524822695034, - "acc_stderr": 0.026129572527180848, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.2695035460992908, + "acc_stderr": 0.026469036818590634, + "brier_score": 0.7560126391237888, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_college_physics": { - "acc": 0.21568627450980393, - "acc_stderr": 0.04092563958237654, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.2647058823529412, + "acc_stderr": 0.043898699568087785, + "brier_score": 0.7657096561986334, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_high_school_physics": { - "acc": 0.2119205298013245, - "acc_stderr": 0.03336767086567977, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.23178807947019867, + "acc_stderr": 0.034454062719870546, + "brier_score": 0.7666365700704936, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_high_school_biology": { - "acc": 0.22258064516129034, - "acc_stderr": 0.023664216671642518, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.1967741935483871, + "acc_stderr": 0.02261640942074203, + "brier_score": 0.7657881483222735, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_college_biology": { - "acc": 0.2222222222222222, - "acc_stderr": 0.03476590104304134, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.2152777777777778, + "acc_stderr": 0.03437079344106135, + "brier_score": 0.7582642369634256, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_anatomy": { - "acc": 0.3111111111111111, - "acc_stderr": 0.03999262876617723, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.21481481481481482, + "acc_stderr": 0.035478541985608236, + "brier_score": 0.7615635040953493, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_college_chemistry": { - "acc": 0.15, - "acc_stderr": 0.03588702812826369, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.14, + "acc_stderr": 0.034873508801977704, + "brier_score": 0.7852103443073895, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_computer_security": { - "acc": 0.23, - "acc_stderr": 0.042295258468165065, - "timestamp": "2024-11-10T15-21-10.938137" - }, - "mmlu_college_computer_science": { "acc": 0.3, "acc_stderr": 0.046056618647183814, - "timestamp": "2024-11-10T15-21-10.938137" + "brier_score": 0.7376635177692521, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" + }, + "mmlu_college_computer_science": { + "acc": 0.21, + "acc_stderr": 0.04093601807403326, + "brier_score": 0.7581582435991463, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_astronomy": { - "acc": 0.23026315789473684, - "acc_stderr": 0.03426059424403165, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.21710526315789475, + "acc_stderr": 0.03355045304882921, + "brier_score": 0.7593982714223672, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_college_mathematics": { - "acc": 0.29, - "acc_stderr": 0.045604802157206845, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.24, + "acc_stderr": 0.042923469599092816, + "brier_score": 0.7655389360560707, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_conceptual_physics": { - "acc": 0.3191489361702128, - "acc_stderr": 0.030472973363380056, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.2936170212765957, + "acc_stderr": 0.02977164271249123, + "brier_score": 0.74895512873291, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_abstract_algebra": { - "acc": 0.33, - "acc_stderr": 0.04725815626252605, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.23, + "acc_stderr": 0.042295258468165065, + "brier_score": 0.7620591635964997, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_high_school_computer_science": { - "acc": 0.34, - "acc_stderr": 0.047609522856952365, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 0.7487049269340801, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_machine_learning": { "acc": 0.22321428571428573, "acc_stderr": 0.03952301967702511, - "timestamp": "2024-11-10T15-21-10.938137" + "brier_score": 0.7577644964672933, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_high_school_chemistry": { - "acc": 0.29064039408866993, - "acc_stderr": 0.03194740072265541, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.22660098522167488, + "acc_stderr": 0.029454863835292965, + "brier_score": 0.7634815544978084, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_high_school_statistics": { - "acc": 0.4722222222222222, - "acc_stderr": 0.0340470532865388, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.2037037037037037, + "acc_stderr": 0.027467401804057986, + "brier_score": 0.7579563734578977, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_elementary_mathematics": { - "acc": 0.26455026455026454, - "acc_stderr": 0.02271746789770862, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.23544973544973544, + "acc_stderr": 0.021851509822031715, + "brier_score": 0.7556070477970247, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_electrical_engineering": { - "acc": 0.2689655172413793, - "acc_stderr": 0.03695183311650232, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.2482758620689655, + "acc_stderr": 0.036001056927277696, + "brier_score": 0.765478714407733, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "mmlu_high_school_mathematics": { - "acc": 0.2814814814814815, - "acc_stderr": 0.027420019350945273, - "timestamp": "2024-11-10T15-21-10.938137" + "acc": 0.3, + "acc_stderr": 0.02794045713622841, + "brier_score": 0.7494715399118812, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-04-57.693669" }, "arc_challenge": { "acc": 0.3651877133105802, diff --git a/metadata/model_scores/results_IEITYuan__Yuan2-2B-hf.json b/metadata/model_scores/results_IEITYuan__Yuan2-2B-hf.json index 042911a..b17531c 100644 --- a/metadata/model_scores/results_IEITYuan__Yuan2-2B-hf.json +++ b/metadata/model_scores/results_IEITYuan__Yuan2-2B-hf.json @@ -1,6 +1,6 @@ { "model_name": "IEITYuan/Yuan2-2B-hf", - "last_updated": "2024-12-04 11:25:43.293918", + "last_updated": "2024-12-19 13:41:57.237772", "results": { "harness": { "minerva_math_precalc": { @@ -228,289 +228,403 @@ "timestamp": "2024-06-14T06-11-06.726003" }, "mmlu_world_religions": { - "acc": 0.21052631578947367, - "acc_stderr": 0.031267817146631786, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.2807017543859649, + "acc_stderr": 0.03446296217088426, + "brier_score": 0.8096009779409294, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_formal_logic": { - "acc": 0.20634920634920634, - "acc_stderr": 0.036196045241242536, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.21428571428571427, + "acc_stderr": 0.03670066451047181, + "brier_score": 0.8492333490932678, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_prehistory": { - "acc": 0.2716049382716049, - "acc_stderr": 0.024748624490537382, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.22530864197530864, + "acc_stderr": 0.023246202647819746, + "brier_score": 0.8711652302179635, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_moral_scenarios": { - "acc": 0.2659217877094972, - "acc_stderr": 0.014776765066438892, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.25027932960893856, + "acc_stderr": 0.014487500852850407, + "brier_score": 0.7841230310531501, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_high_school_world_history": { "acc": 0.26582278481012656, - "acc_stderr": 0.02875679962965834, - "timestamp": "2024-11-10T17-48-44.142080" + "acc_stderr": 0.028756799629658335, + "brier_score": 0.8327831803880661, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_moral_disputes": { - "acc": 0.27167630057803466, - "acc_stderr": 0.023948512905468348, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.2514450867052023, + "acc_stderr": 0.023357365785874037, + "brier_score": 0.856367471910325, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_professional_law": { - "acc": 0.2666232073011734, - "acc_stderr": 0.01129383603161214, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.22946544980443284, + "acc_stderr": 0.010739489382279505, + "brier_score": 0.8877477808741326, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_logical_fallacies": { - "acc": 0.2147239263803681, - "acc_stderr": 0.03226219377286774, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.2085889570552147, + "acc_stderr": 0.031921934489347235, + "brier_score": 0.8614073601523989, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_high_school_us_history": { - "acc": 0.21568627450980393, - "acc_stderr": 0.028867431449849303, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.25, + "acc_stderr": 0.03039153369274154, + "brier_score": 0.8723475480948988, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_philosophy": { - "acc": 0.2540192926045016, - "acc_stderr": 0.024723861504771693, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.2282958199356913, + "acc_stderr": 0.02383930331139821, + "brier_score": 0.873744048004299, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_jurisprudence": { - "acc": 0.23148148148148148, - "acc_stderr": 0.04077494709252627, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.26851851851851855, + "acc_stderr": 0.04284467968052191, + "brier_score": 0.8528512268750248, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_international_law": { - "acc": 0.24793388429752067, - "acc_stderr": 0.03941897526516302, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.21487603305785125, + "acc_stderr": 0.03749492448709699, + "brier_score": 0.8545314075336393, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_high_school_european_history": { - "acc": 0.24848484848484848, - "acc_stderr": 0.03374402644139405, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.23636363636363636, + "acc_stderr": 0.033175059300091805, + "brier_score": 0.8557986356963414, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_high_school_government_and_politics": { - "acc": 0.2849740932642487, - "acc_stderr": 0.032577140777096614, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.20725388601036268, + "acc_stderr": 0.029252823291803638, + "brier_score": 0.915030398053468, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_high_school_microeconomics": { - "acc": 0.2184873949579832, - "acc_stderr": 0.026841514322958934, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.23529411764705882, + "acc_stderr": 0.027553614467863807, + "brier_score": 0.8578690440470526, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_high_school_geography": { - "acc": 0.22727272727272727, - "acc_stderr": 0.029857515673386396, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.21717171717171718, + "acc_stderr": 0.02937661648494563, + "brier_score": 0.9057046642331704, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_high_school_psychology": { - "acc": 0.21651376146788992, - "acc_stderr": 0.017658710594443138, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.22385321100917432, + "acc_stderr": 0.01787121776779021, + "brier_score": 0.8527920303028815, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_public_relations": { - "acc": 0.24545454545454545, - "acc_stderr": 0.04122066502878284, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.2545454545454545, + "acc_stderr": 0.04172343038705383, + "brier_score": 0.8259331453627159, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_us_foreign_policy": { - "acc": 0.32, - "acc_stderr": 0.04688261722621505, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.36, + "acc_stderr": 0.04824181513244218, + "brier_score": 0.7771535619076206, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_sociology": { "acc": 0.2537313432835821, - "acc_stderr": 0.030769444967296014, - "timestamp": "2024-11-10T17-48-44.142080" + "acc_stderr": 0.030769444967296035, + "brier_score": 0.8309135703592723, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_high_school_macroeconomics": { - "acc": 0.2230769230769231, - "acc_stderr": 0.021107730127244, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.2282051282051282, + "acc_stderr": 0.02127839386358628, + "brier_score": 0.8713082619742111, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_security_studies": { - "acc": 0.22040816326530613, - "acc_stderr": 0.026537045312145298, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.22857142857142856, + "acc_stderr": 0.026882144922307744, + "brier_score": 0.8254330669999973, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_professional_psychology": { - "acc": 0.26633986928104575, - "acc_stderr": 0.0178831881346672, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.25326797385620914, + "acc_stderr": 0.017593486895366835, + "brier_score": 0.8505259159401877, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_human_sexuality": { - "acc": 0.22900763358778625, - "acc_stderr": 0.036853466317118506, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.3053435114503817, + "acc_stderr": 0.04039314978724562, + "brier_score": 0.8192683078241423, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_econometrics": { - "acc": 0.2631578947368421, - "acc_stderr": 0.04142439719489362, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.20175438596491227, + "acc_stderr": 0.037752050135836386, + "brier_score": 0.8538722129937036, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_miscellaneous": { - "acc": 0.2796934865900383, - "acc_stderr": 0.016050792148036546, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.23243933588761176, + "acc_stderr": 0.015104550008905687, + "brier_score": 0.8194701596707327, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_marketing": { - "acc": 0.23504273504273504, - "acc_stderr": 0.027778835904935437, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.2863247863247863, + "acc_stderr": 0.02961432369045665, + "brier_score": 0.8463996248482538, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_management": { - "acc": 0.2621359223300971, - "acc_stderr": 0.04354631077260595, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.22330097087378642, + "acc_stderr": 0.04123553189891431, + "brier_score": 0.8491363029476184, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_nutrition": { - "acc": 0.238562091503268, - "acc_stderr": 0.02440439492808787, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.23529411764705882, + "acc_stderr": 0.024288619466046105, + "brier_score": 0.857691953221437, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_medical_genetics": { - "acc": 0.23, - "acc_stderr": 0.04229525846816506, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.33, + "acc_stderr": 0.04725815626252604, + "brier_score": 0.7921112092000784, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_human_aging": { - "acc": 0.2914798206278027, - "acc_stderr": 0.030500283176545906, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.29596412556053814, + "acc_stderr": 0.03063659134869981, + "brier_score": 0.7785455242696859, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_professional_medicine": { - "acc": 0.18382352941176472, - "acc_stderr": 0.023529242185193106, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.19117647058823528, + "acc_stderr": 0.02388688192244034, + "brier_score": 0.9392371353631491, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_college_medicine": { - "acc": 0.23699421965317918, - "acc_stderr": 0.03242414757483098, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.2543352601156069, + "acc_stderr": 0.0332055644308557, + "brier_score": 0.8467048802466525, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_business_ethics": { - "acc": 0.25, - "acc_stderr": 0.04351941398892446, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.18, + "acc_stderr": 0.03861229196653697, + "brier_score": 0.8523128729212764, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_clinical_knowledge": { - "acc": 0.29056603773584905, - "acc_stderr": 0.02794321998933716, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.2490566037735849, + "acc_stderr": 0.026616482980501704, + "brier_score": 0.8344599422751379, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_global_facts": { - "acc": 0.33, - "acc_stderr": 0.04725815626252605, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.2, + "acc_stderr": 0.04020151261036846, + "brier_score": 0.8578052760213979, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_virology": { - "acc": 0.3132530120481928, - "acc_stderr": 0.03610805018031023, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.2469879518072289, + "acc_stderr": 0.03357351982064536, + "brier_score": 0.8164458967902466, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_professional_accounting": { - "acc": 0.25177304964539005, - "acc_stderr": 0.025892151156709405, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.21631205673758866, + "acc_stderr": 0.02456172056056277, + "brier_score": 0.8717352507015241, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_college_physics": { - "acc": 0.22549019607843138, - "acc_stderr": 0.041583075330832865, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.24509803921568626, + "acc_stderr": 0.04280105837364395, + "brier_score": 0.8500361889495441, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_high_school_physics": { "acc": 0.2052980132450331, "acc_stderr": 0.03297986648473835, - "timestamp": "2024-11-10T17-48-44.142080" + "brier_score": 0.9160716728274827, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_high_school_biology": { - "acc": 0.2, - "acc_stderr": 0.02275520495954294, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.2161290322580645, + "acc_stderr": 0.023415293433568532, + "brier_score": 0.9022114625100559, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_college_biology": { - "acc": 0.19444444444444445, - "acc_stderr": 0.03309615177059006, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.2361111111111111, + "acc_stderr": 0.03551446610810826, + "brier_score": 0.8826057166424514, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_anatomy": { - "acc": 0.23703703703703705, - "acc_stderr": 0.03673731683969506, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.22962962962962963, + "acc_stderr": 0.03633384414073465, + "brier_score": 0.8435118363521027, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_college_chemistry": { - "acc": 0.22, - "acc_stderr": 0.04163331998932269, - "timestamp": "2024-11-10T17-48-44.142080" - }, - "mmlu_computer_security": { "acc": 0.29, "acc_stderr": 0.045604802157206845, - "timestamp": "2024-11-10T17-48-44.142080" + "brier_score": 0.8001169174103776, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" + }, + "mmlu_computer_security": { + "acc": 0.26, + "acc_stderr": 0.04408440022768078, + "brier_score": 0.8297649101265755, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_college_computer_science": { - "acc": 0.15, - "acc_stderr": 0.0358870281282637, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "brier_score": 0.8745706761974686, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_astronomy": { - "acc": 0.18421052631578946, - "acc_stderr": 0.031546980450822305, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.23026315789473684, + "acc_stderr": 0.03426059424403165, + "brier_score": 0.8320305956799038, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_college_mathematics": { - "acc": 0.25, - "acc_stderr": 0.04351941398892446, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.28, + "acc_stderr": 0.04512608598542127, + "brier_score": 0.7980341338123739, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_conceptual_physics": { - "acc": 0.3191489361702128, - "acc_stderr": 0.030472973363380052, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.2723404255319149, + "acc_stderr": 0.0291012906983867, + "brier_score": 0.7987215599898191, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_abstract_algebra": { - "acc": 0.22, - "acc_stderr": 0.041633319989322695, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.23, + "acc_stderr": 0.04229525846816506, + "brier_score": 0.7936128840334643, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_high_school_computer_science": { - "acc": 0.25, - "acc_stderr": 0.04351941398892446, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.2, + "acc_stderr": 0.04020151261036845, + "brier_score": 0.8855977049559729, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_machine_learning": { - "acc": 0.2857142857142857, - "acc_stderr": 0.04287858751340456, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.25, + "acc_stderr": 0.04109974682633932, + "brier_score": 0.833709833107949, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_high_school_chemistry": { - "acc": 0.2413793103448276, - "acc_stderr": 0.030108330718011625, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.19704433497536947, + "acc_stderr": 0.02798672466673622, + "brier_score": 0.8658986710939709, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_high_school_statistics": { - "acc": 0.33796296296296297, - "acc_stderr": 0.032259413526312945, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.22685185185185186, + "acc_stderr": 0.02856165010242226, + "brier_score": 0.8810555037720272, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_elementary_mathematics": { - "acc": 0.25396825396825395, - "acc_stderr": 0.022418042891113946, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.21957671957671956, + "acc_stderr": 0.021320018599770355, + "brier_score": 0.8219060988248941, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_electrical_engineering": { - "acc": 0.30344827586206896, - "acc_stderr": 0.038312260488503336, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.2620689655172414, + "acc_stderr": 0.036646663372252565, + "brier_score": 0.831102037949681, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "mmlu_high_school_mathematics": { - "acc": 0.24074074074074073, - "acc_stderr": 0.026067159222275788, - "timestamp": "2024-11-10T17-48-44.142080" + "acc": 0.2777777777777778, + "acc_stderr": 0.02730914058823017, + "brier_score": 0.8027807301116876, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-52-37.408897" }, "arc_challenge": { "acc": 0.19880546075085323, diff --git a/metadata/model_scores/results_NinedayWang__PolyCoder-2.7B.json b/metadata/model_scores/results_NinedayWang__PolyCoder-2.7B.json index 32cda3b..b3a7b42 100644 --- a/metadata/model_scores/results_NinedayWang__PolyCoder-2.7B.json +++ b/metadata/model_scores/results_NinedayWang__PolyCoder-2.7B.json @@ -1,292 +1,406 @@ { "model_name": "NinedayWang__PolyCoder-2.7B", - "last_updated": "2024-12-04 11:22:36.527352", + "last_updated": "2024-12-19 13:37:53.735592", "results": { "harness": { "mmlu_world_religions": { - "acc": 0.3157894736842105, - "acc_stderr": 0.03565079670708312, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.30409356725146197, + "acc_stderr": 0.0352821125824523, + "brier_score": 0.8938431035787754, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_formal_logic": { - "acc": 0.18253968253968253, - "acc_stderr": 0.03455071019102147, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.2857142857142857, + "acc_stderr": 0.04040610178208841, + "brier_score": 1.0150480465735368, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_prehistory": { "acc": 0.2191358024691358, - "acc_stderr": 0.023016705640262196, - "timestamp": "2024-11-22T00-21-02.759683" + "acc_stderr": 0.023016705640262203, + "brier_score": 1.0688599538686459, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_moral_scenarios": { - "acc": 0.24804469273743016, - "acc_stderr": 0.014444157808261436, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.23798882681564246, + "acc_stderr": 0.014242630070574885, + "brier_score": 0.8886120543076361, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_high_school_world_history": { - "acc": 0.2742616033755274, - "acc_stderr": 0.02904133351059802, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.26582278481012656, + "acc_stderr": 0.028756799629658335, + "brier_score": 0.9960394039300178, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_moral_disputes": { - "acc": 0.23699421965317918, - "acc_stderr": 0.02289408248992599, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.2514450867052023, + "acc_stderr": 0.023357365785874037, + "brier_score": 1.05020278137478, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_professional_law": { - "acc": 0.24771838331160365, - "acc_stderr": 0.011025499291443737, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.2457627118644068, + "acc_stderr": 0.01099615663514269, + "brier_score": 1.1471446337073656, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_logical_fallacies": { - "acc": 0.27607361963190186, - "acc_stderr": 0.0351238528370505, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.22085889570552147, + "acc_stderr": 0.032591773927421776, + "brier_score": 1.1352476522307973, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_high_school_us_history": { - "acc": 0.2647058823529412, - "acc_stderr": 0.03096451792692341, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.25, + "acc_stderr": 0.03039153369274154, + "brier_score": 1.0396970803983272, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_philosophy": { - "acc": 0.26366559485530544, - "acc_stderr": 0.02502553850053234, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.1832797427652733, + "acc_stderr": 0.0219741988482658, + "brier_score": 1.1221353849032953, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_jurisprudence": { - "acc": 0.21296296296296297, - "acc_stderr": 0.03957835471980978, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.25, + "acc_stderr": 0.04186091791394607, + "brier_score": 1.088399168511025, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_international_law": { - "acc": 0.3140495867768595, - "acc_stderr": 0.04236964753041019, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.2396694214876033, + "acc_stderr": 0.03896878985070417, + "brier_score": 1.126887944187717, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_high_school_european_history": { - "acc": 0.2606060606060606, - "acc_stderr": 0.03427743175816524, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.21818181818181817, + "acc_stderr": 0.03225078108306289, + "brier_score": 0.9985889584374511, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_high_school_government_and_politics": { - "acc": 0.31088082901554404, - "acc_stderr": 0.03340361906276585, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.20207253886010362, + "acc_stderr": 0.02897908979429673, + "brier_score": 1.161175617365508, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_high_school_microeconomics": { - "acc": 0.23529411764705882, - "acc_stderr": 0.027553614467863804, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.20588235294117646, + "acc_stderr": 0.026265024608275886, + "brier_score": 1.136253757095844, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_high_school_geography": { - "acc": 0.2727272727272727, - "acc_stderr": 0.03173071239071724, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.17676767676767677, + "acc_stderr": 0.027178752639044915, + "brier_score": 1.1626575476275403, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_high_school_psychology": { - "acc": 0.20733944954128442, - "acc_stderr": 0.017381415563608674, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.1908256880733945, + "acc_stderr": 0.01684767640009109, + "brier_score": 1.1522750595179523, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_public_relations": { - "acc": 0.22727272727272727, - "acc_stderr": 0.04013964554072773, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.21818181818181817, + "acc_stderr": 0.03955932861795833, + "brier_score": 1.1047895125095324, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_us_foreign_policy": { - "acc": 0.24, - "acc_stderr": 0.042923469599092816, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.29, + "acc_stderr": 0.045604802157206845, + "brier_score": 0.9752901097938361, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_sociology": { - "acc": 0.22388059701492538, - "acc_stderr": 0.029475250236017193, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.23880597014925373, + "acc_stderr": 0.030147775935409227, + "brier_score": 1.106775743949372, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_high_school_macroeconomics": { - "acc": 0.34615384615384615, - "acc_stderr": 0.024121125416941183, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.20256410256410257, + "acc_stderr": 0.020377660970371397, + "brier_score": 1.106854573740813, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_security_studies": { "acc": 0.1836734693877551, "acc_stderr": 0.024789071332007643, - "timestamp": "2024-11-22T00-21-02.759683" + "brier_score": 1.2265616632194247, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_professional_psychology": { - "acc": 0.24509803921568626, - "acc_stderr": 0.017401816711427657, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.25163398692810457, + "acc_stderr": 0.01755581809132225, + "brier_score": 1.1237729121517326, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_human_sexuality": { "acc": 0.2595419847328244, - "acc_stderr": 0.03844876139785271, - "timestamp": "2024-11-22T00-21-02.759683" + "acc_stderr": 0.0384487613978527, + "brier_score": 1.0527196659699807, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_econometrics": { "acc": 0.23684210526315788, - "acc_stderr": 0.03999423879281335, - "timestamp": "2024-11-22T00-21-02.759683" + "acc_stderr": 0.039994238792813386, + "brier_score": 1.0186789892109231, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_miscellaneous": { - "acc": 0.25287356321839083, - "acc_stderr": 0.015543377313719681, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.24010217113665389, + "acc_stderr": 0.015274685213734188, + "brier_score": 0.9769991448277808, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_marketing": { - "acc": 0.1794871794871795, - "acc_stderr": 0.02514093595033545, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.2905982905982906, + "acc_stderr": 0.029745048572674057, + "brier_score": 1.102306778717253, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_management": { - "acc": 0.13592233009708737, - "acc_stderr": 0.033932957297610145, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.17475728155339806, + "acc_stderr": 0.03760178006026621, + "brier_score": 1.190507413382486, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_nutrition": { - "acc": 0.23529411764705882, - "acc_stderr": 0.024288619466046105, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.21568627450980393, + "acc_stderr": 0.02355083135199509, + "brier_score": 1.0773290847012829, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_medical_genetics": { - "acc": 0.31, - "acc_stderr": 0.04648231987117316, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "brier_score": 0.9521524003092359, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_human_aging": { "acc": 0.31390134529147984, - "acc_stderr": 0.031146796482972465, - "timestamp": "2024-11-22T00-21-02.759683" + "acc_stderr": 0.03114679648297246, + "brier_score": 0.947392555723536, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_professional_medicine": { - "acc": 0.4411764705882353, - "acc_stderr": 0.0301619119307671, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.18382352941176472, + "acc_stderr": 0.02352924218519311, + "brier_score": 1.1710175839535208, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_college_medicine": { - "acc": 0.23121387283236994, - "acc_stderr": 0.032147373020294696, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.2138728323699422, + "acc_stderr": 0.03126511206173043, + "brier_score": 1.1487105628511547, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_business_ethics": { - "acc": 0.22, - "acc_stderr": 0.041633319989322695, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "brier_score": 0.9736715265758287, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_clinical_knowledge": { - "acc": 0.2641509433962264, - "acc_stderr": 0.027134291628741702, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.21132075471698114, + "acc_stderr": 0.025125766484827856, + "brier_score": 1.101333058881401, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_global_facts": { "acc": 0.18, "acc_stderr": 0.038612291966536955, - "timestamp": "2024-11-22T00-21-02.759683" + "brier_score": 1.0434812793332202, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_virology": { - "acc": 0.1746987951807229, - "acc_stderr": 0.029560326211256822, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.28313253012048195, + "acc_stderr": 0.03507295431370518, + "brier_score": 1.0022878477677757, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_professional_accounting": { - "acc": 0.26595744680851063, - "acc_stderr": 0.026358065698880592, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.22695035460992907, + "acc_stderr": 0.024987106365642976, + "brier_score": 1.1000058041634257, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_college_physics": { "acc": 0.21568627450980393, "acc_stderr": 0.040925639582376556, - "timestamp": "2024-11-22T00-21-02.759683" + "brier_score": 1.0905135942657984, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_high_school_physics": { - "acc": 0.2781456953642384, - "acc_stderr": 0.03658603262763743, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.2052980132450331, + "acc_stderr": 0.032979866484738336, + "brier_score": 1.1401240382655313, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_high_school_biology": { - "acc": 0.3032258064516129, - "acc_stderr": 0.02614868593067175, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.19032258064516128, + "acc_stderr": 0.022331707611823085, + "brier_score": 1.119124794185339, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_college_biology": { - "acc": 0.2569444444444444, - "acc_stderr": 0.03653946969442099, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.25, + "acc_stderr": 0.03621034121889507, + "brier_score": 1.0291517128501562, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_anatomy": { - "acc": 0.3333333333333333, - "acc_stderr": 0.04072314811876837, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.18518518518518517, + "acc_stderr": 0.03355677216313142, + "brier_score": 1.1034100401415832, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_college_chemistry": { "acc": 0.19, - "acc_stderr": 0.039427724440366234, - "timestamp": "2024-11-22T00-21-02.759683" + "acc_stderr": 0.03942772444036623, + "brier_score": 1.0940351459391089, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_computer_security": { "acc": 0.28, - "acc_stderr": 0.04512608598542127, - "timestamp": "2024-11-22T00-21-02.759683" + "acc_stderr": 0.045126085985421276, + "brier_score": 1.0316655764682736, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_college_computer_science": { - "acc": 0.34, - "acc_stderr": 0.04760952285695236, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 1.1385161111574733, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_astronomy": { - "acc": 0.19078947368421054, - "acc_stderr": 0.031975658210325, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.17763157894736842, + "acc_stderr": 0.031103182383123398, + "brier_score": 1.15340863761547, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_college_mathematics": { - "acc": 0.27, - "acc_stderr": 0.044619604333847394, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "brier_score": 1.0943448594442058, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_conceptual_physics": { - "acc": 0.2170212765957447, - "acc_stderr": 0.02694748312149622, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.26382978723404255, + "acc_stderr": 0.02880998985410298, + "brier_score": 0.9734414312468421, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_abstract_algebra": { - "acc": 0.24, - "acc_stderr": 0.04292346959909283, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.22, + "acc_stderr": 0.04163331998932269, + "brier_score": 1.0547517500487764, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_high_school_computer_science": { - "acc": 0.3, - "acc_stderr": 0.04605661864718381, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 1.096824166644259, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_machine_learning": { - "acc": 0.25892857142857145, - "acc_stderr": 0.04157751539865629, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.32142857142857145, + "acc_stderr": 0.04432804055291519, + "brier_score": 0.955715696475034, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_high_school_chemistry": { - "acc": 0.270935960591133, - "acc_stderr": 0.031270907132976984, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.17733990147783252, + "acc_stderr": 0.02687433727680835, + "brier_score": 1.084687793483445, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_high_school_statistics": { - "acc": 0.38425925925925924, - "acc_stderr": 0.03317354514310742, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.1527777777777778, + "acc_stderr": 0.02453632602613422, + "brier_score": 1.2395721614530157, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_elementary_mathematics": { - "acc": 0.2222222222222222, - "acc_stderr": 0.021411684393694196, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.20899470899470898, + "acc_stderr": 0.020940481565334835, + "brier_score": 1.0251953695947718, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_electrical_engineering": { - "acc": 0.22758620689655173, - "acc_stderr": 0.03493950380131184, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.2413793103448276, + "acc_stderr": 0.03565998174135302, + "brier_score": 1.061490829841905, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "mmlu_high_school_mathematics": { - "acc": 0.25555555555555554, - "acc_stderr": 0.02659393910184407, - "timestamp": "2024-11-22T00-21-02.759683" + "acc": 0.2074074074074074, + "acc_stderr": 0.024720713193952165, + "brier_score": 1.0396806234134943, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-53-22.386081" }, "arc_challenge": { "acc": 0.181740614334471, diff --git a/metadata/model_scores/results_Qwen__Qwen1.5-14B.json b/metadata/model_scores/results_Qwen__Qwen1.5-14B.json index 988bed5..3af3c6d 100644 --- a/metadata/model_scores/results_Qwen__Qwen1.5-14B.json +++ b/metadata/model_scores/results_Qwen__Qwen1.5-14B.json @@ -1,292 +1,406 @@ { "model_name": "Qwen__Qwen1.5-14B", - "last_updated": "2024-12-04 11:25:20.383414", + "last_updated": "2024-12-19 13:41:26.474331", "results": { "harness": { "mmlu_world_religions": { - "acc": 0.8362573099415205, - "acc_stderr": 0.028380919596145866, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.8187134502923976, + "acc_stderr": 0.02954774168764004, + "brier_score": 0.2610748216249862, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_formal_logic": { - "acc": 0.5476190476190477, - "acc_stderr": 0.044518079590553275, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.5079365079365079, + "acc_stderr": 0.044715725362943486, + "brier_score": 0.6157905467597715, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_prehistory": { - "acc": 0.7129629629629629, - "acc_stderr": 0.02517104191530968, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.7191358024691358, + "acc_stderr": 0.02500646975579921, + "brier_score": 0.3904460017679024, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_moral_scenarios": { - "acc": 0.4424581005586592, - "acc_stderr": 0.016611393687268577, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.3094972067039106, + "acc_stderr": 0.015461169002371551, + "brier_score": 0.8483799669259762, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_high_school_world_history": { - "acc": 0.8312236286919831, - "acc_stderr": 0.024381406832586223, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.8396624472573839, + "acc_stderr": 0.023884380925965676, + "brier_score": 0.23825771259083467, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_moral_disputes": { - "acc": 0.7456647398843931, - "acc_stderr": 0.023445826276545546, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.7543352601156069, + "acc_stderr": 0.02317629820399201, + "brier_score": 0.38253639395673245, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_professional_law": { - "acc": 0.485006518904824, - "acc_stderr": 0.012764493202193257, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.4817470664928292, + "acc_stderr": 0.012761723960595472, + "brier_score": 0.6508731202105119, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_logical_fallacies": { - "acc": 0.7668711656441718, - "acc_stderr": 0.0332201579577674, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.7914110429447853, + "acc_stderr": 0.03192193448934725, + "brier_score": 0.30933584311149803, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_high_school_us_history": { "acc": 0.8137254901960784, - "acc_stderr": 0.027325470966716312, - "timestamp": "2024-11-25T22-36-01.015934" + "acc_stderr": 0.02732547096671632, + "brier_score": 0.27448796132676617, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_philosophy": { - "acc": 0.7202572347266881, - "acc_stderr": 0.0254942593506949, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.6881028938906752, + "acc_stderr": 0.026311858071854155, + "brier_score": 0.3925745785990191, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_jurisprudence": { - "acc": 0.75, - "acc_stderr": 0.04186091791394607, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.7222222222222222, + "acc_stderr": 0.04330043749650742, + "brier_score": 0.35487065011938335, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_international_law": { - "acc": 0.8429752066115702, - "acc_stderr": 0.0332124484254713, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.8099173553719008, + "acc_stderr": 0.035817969517092825, + "brier_score": 0.293423952055567, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_high_school_european_history": { - "acc": 0.8424242424242424, - "acc_stderr": 0.02845038880528436, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.8181818181818182, + "acc_stderr": 0.030117688929503564, + "brier_score": 0.26603788895394054, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_high_school_government_and_politics": { - "acc": 0.8963730569948186, - "acc_stderr": 0.02199531196364424, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.8601036269430051, + "acc_stderr": 0.025033870583015174, + "brier_score": 0.18537190762315367, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_high_school_microeconomics": { - "acc": 0.7605042016806722, - "acc_stderr": 0.02772206549336127, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.7310924369747899, + "acc_stderr": 0.028801392193631276, + "brier_score": 0.3764555181573261, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_high_school_geography": { - "acc": 0.8686868686868687, - "acc_stderr": 0.024063156416822523, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.8383838383838383, + "acc_stderr": 0.02622591986362929, + "brier_score": 0.23701753590451613, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_high_school_psychology": { - "acc": 0.8642201834862385, - "acc_stderr": 0.014686907556340001, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.8403669724770643, + "acc_stderr": 0.015703498348461777, + "brier_score": 0.22737277694603805, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_public_relations": { - "acc": 0.6454545454545455, - "acc_stderr": 0.04582004841505417, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.6636363636363637, + "acc_stderr": 0.04525393596302505, + "brier_score": 0.45027583895559403, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_us_foreign_policy": { - "acc": 0.88, - "acc_stderr": 0.03265986323710906, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.87, + "acc_stderr": 0.03379976689896309, + "brier_score": 0.2072955878159843, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_sociology": { - "acc": 0.8407960199004975, - "acc_stderr": 0.02587064676616913, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.8208955223880597, + "acc_stderr": 0.027113286753111848, + "brier_score": 0.25116180826584866, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_high_school_macroeconomics": { - "acc": 0.7333333333333333, - "acc_stderr": 0.02242127361292371, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.7051282051282052, + "acc_stderr": 0.02311936275823229, + "brier_score": 0.41660313734183246, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_security_studies": { - "acc": 0.8040816326530612, - "acc_stderr": 0.025409301953225678, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.7673469387755102, + "acc_stderr": 0.02704925791589618, + "brier_score": 0.3078435560741998, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_professional_psychology": { - "acc": 0.704248366013072, - "acc_stderr": 0.018463154132632813, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.6764705882352942, + "acc_stderr": 0.018926082916083383, + "brier_score": 0.4253943613385468, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_human_sexuality": { "acc": 0.7633587786259542, - "acc_stderr": 0.03727673575596915, - "timestamp": "2024-11-25T22-36-01.015934" + "acc_stderr": 0.03727673575596914, + "brier_score": 0.3292387581931091, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_econometrics": { "acc": 0.5526315789473685, - "acc_stderr": 0.046774730044911984, - "timestamp": "2024-11-25T22-36-01.015934" + "acc_stderr": 0.04677473004491199, + "brier_score": 0.5716568770804077, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_miscellaneous": { - "acc": 0.8378033205619413, - "acc_stderr": 0.013182222616720876, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.8390804597701149, + "acc_stderr": 0.013140225515611729, + "brier_score": 0.2219740214324536, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_marketing": { - "acc": 0.8675213675213675, - "acc_stderr": 0.022209309073165612, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.9017094017094017, + "acc_stderr": 0.019503444900757567, + "brier_score": 0.15146154108163973, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_management": { - "acc": 0.8058252427184466, - "acc_stderr": 0.039166677628225836, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.8155339805825242, + "acc_stderr": 0.03840423627288276, + "brier_score": 0.26640579362245564, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_nutrition": { - "acc": 0.7483660130718954, - "acc_stderr": 0.024848018263875202, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.761437908496732, + "acc_stderr": 0.024404394928087873, + "brier_score": 0.33426490557607696, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_medical_genetics": { "acc": 0.79, - "acc_stderr": 0.04093601807403326, - "timestamp": "2024-11-25T22-36-01.015934" + "acc_stderr": 0.040936018074033256, + "brier_score": 0.3197741302501219, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_human_aging": { - "acc": 0.7309417040358744, - "acc_stderr": 0.029763779406874975, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.7130044843049327, + "acc_stderr": 0.030360379710291954, + "brier_score": 0.3881359337304798, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_professional_medicine": { - "acc": 0.7205882352941176, - "acc_stderr": 0.027257202606114944, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.7316176470588235, + "acc_stderr": 0.026917481224377218, + "brier_score": 0.3969042642744269, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_college_medicine": { - "acc": 0.6936416184971098, - "acc_stderr": 0.035149425512674394, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.6763005780346821, + "acc_stderr": 0.0356760379963917, + "brier_score": 0.41220937602784485, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_business_ethics": { - "acc": 0.76, - "acc_stderr": 0.04292346959909282, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.73, + "acc_stderr": 0.044619604333847394, + "brier_score": 0.334739258950162, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_clinical_knowledge": { "acc": 0.7471698113207547, "acc_stderr": 0.026749899771241214, - "timestamp": "2024-11-25T22-36-01.015934" + "brier_score": 0.3476556369277268, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_global_facts": { - "acc": 0.5, - "acc_stderr": 0.050251890762960605, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.48, + "acc_stderr": 0.050211673156867795, + "brier_score": 0.6611443955064803, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_virology": { - "acc": 0.4578313253012048, - "acc_stderr": 0.0387862677100236, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.46987951807228917, + "acc_stderr": 0.03885425420866767, + "brier_score": 0.7607967763367265, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_professional_accounting": { - "acc": 0.5, - "acc_stderr": 0.029827499313594685, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.5212765957446809, + "acc_stderr": 0.029800481645628693, + "brier_score": 0.6138256593226751, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_college_physics": { "acc": 0.5, "acc_stderr": 0.04975185951049946, - "timestamp": "2024-11-25T22-36-01.015934" + "brier_score": 0.580215867589239, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_high_school_physics": { - "acc": 0.48344370860927155, - "acc_stderr": 0.04080244185628972, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.46357615894039733, + "acc_stderr": 0.04071636065944216, + "brier_score": 0.6646346273273989, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_high_school_biology": { - "acc": 0.8419354838709677, - "acc_stderr": 0.02075283151187525, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.8193548387096774, + "acc_stderr": 0.021886178567172534, + "brier_score": 0.24254150264085433, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_college_biology": { - "acc": 0.7638888888888888, - "acc_stderr": 0.03551446610810826, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.7361111111111112, + "acc_stderr": 0.03685651095897532, + "brier_score": 0.3285251430155265, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_anatomy": { - "acc": 0.6370370370370371, - "acc_stderr": 0.041539484047424, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.6148148148148148, + "acc_stderr": 0.042039210401562783, + "brier_score": 0.4720453165969145, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_college_chemistry": { - "acc": 0.5, - "acc_stderr": 0.050251890762960605, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.54, + "acc_stderr": 0.05009082659620333, + "brier_score": 0.5916111839506865, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_computer_security": { - "acc": 0.8, - "acc_stderr": 0.04020151261036845, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.83, + "acc_stderr": 0.03775251680686371, + "brier_score": 0.26723005206388406, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_college_computer_science": { - "acc": 0.59, - "acc_stderr": 0.04943110704237102, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.54, + "acc_stderr": 0.05009082659620333, + "brier_score": 0.5538877668625505, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_astronomy": { - "acc": 0.7302631578947368, - "acc_stderr": 0.03611780560284898, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.7171052631578947, + "acc_stderr": 0.03665349695640767, + "brier_score": 0.39010894641974114, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_college_mathematics": { - "acc": 0.47, - "acc_stderr": 0.05016135580465919, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.41, + "acc_stderr": 0.04943110704237101, + "brier_score": 0.660017075705441, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_conceptual_physics": { - "acc": 0.7021276595744681, - "acc_stderr": 0.029896145682095462, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.676595744680851, + "acc_stderr": 0.030579442773610344, + "brier_score": 0.41226055884215135, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_abstract_algebra": { - "acc": 0.36, - "acc_stderr": 0.048241815132442176, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.4, + "acc_stderr": 0.04923659639173309, + "brier_score": 0.6865250232090737, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_high_school_computer_science": { - "acc": 0.76, - "acc_stderr": 0.042923469599092816, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.73, + "acc_stderr": 0.04461960433384739, + "brier_score": 0.34862909935613806, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_machine_learning": { - "acc": 0.5446428571428571, - "acc_stderr": 0.04726835553719098, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.48214285714285715, + "acc_stderr": 0.047427623612430116, + "brier_score": 0.568482496898005, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_high_school_chemistry": { - "acc": 0.5911330049261084, - "acc_stderr": 0.03459058815883232, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.625615763546798, + "acc_stderr": 0.03405155380561952, + "brier_score": 0.48655754490312986, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_high_school_statistics": { - "acc": 0.6296296296296297, - "acc_stderr": 0.03293377139415191, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.5555555555555556, + "acc_stderr": 0.03388857118502325, + "brier_score": 0.5375076019475299, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_elementary_mathematics": { - "acc": 0.58994708994709, - "acc_stderr": 0.025331202438944433, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.6058201058201058, + "acc_stderr": 0.025167982333894143, + "brier_score": 0.5186656828914877, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_electrical_engineering": { - "acc": 0.7103448275862069, - "acc_stderr": 0.03780019230438014, - "timestamp": "2024-11-25T22-36-01.015934" + "acc": 0.6551724137931034, + "acc_stderr": 0.03960933549451208, + "brier_score": 0.44258441209465227, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "mmlu_high_school_mathematics": { "acc": 0.4444444444444444, "acc_stderr": 0.03029677128606732, - "timestamp": "2024-11-25T22-36-01.015934" + "brier_score": 0.6408421373455501, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-26-49.303609" }, "arc_challenge": { "acc": 0.5307167235494881, diff --git a/metadata/model_scores/results_Qwen__Qwen2.5-32B.json b/metadata/model_scores/results_Qwen__Qwen2.5-32B.json index 08f684a..6f79c09 100644 --- a/metadata/model_scores/results_Qwen__Qwen2.5-32B.json +++ b/metadata/model_scores/results_Qwen__Qwen2.5-32B.json @@ -1,292 +1,406 @@ { "model_name": "Qwen__Qwen2.5-32B", - "last_updated": "2024-12-04 11:25:05.203085", + "last_updated": "2024-12-19 13:41:07.988448", "results": { "harness": { "mmlu_world_religions": { - "acc": 0.9064327485380117, - "acc_stderr": 0.02233599323116327, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.8830409356725146, + "acc_stderr": 0.02464806896136614, + "brier_score": 0.16792329024471364, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_formal_logic": { - "acc": 0.7222222222222222, - "acc_stderr": 0.040061680838488774, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.7142857142857143, + "acc_stderr": 0.0404061017820884, + "brier_score": 0.37022061759201313, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_prehistory": { "acc": 0.9135802469135802, "acc_stderr": 0.015634305710693557, - "timestamp": "2024-11-28T06-58-27.057743" + "brier_score": 0.13083815199755516, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_moral_scenarios": { - "acc": 0.7541899441340782, - "acc_stderr": 0.014400296429225613, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.506145251396648, + "acc_stderr": 0.016721238483631416, + "brier_score": 0.7021992993666557, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_high_school_world_history": { - "acc": 0.9240506329113924, - "acc_stderr": 0.0172446332510657, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.9071729957805907, + "acc_stderr": 0.018889750550956715, + "brier_score": 0.12776535151245633, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_moral_disputes": { - "acc": 0.8439306358381503, - "acc_stderr": 0.019539014685374036, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.8554913294797688, + "acc_stderr": 0.01892976451346872, + "brier_score": 0.225007356758669, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_professional_law": { - "acc": 0.6492829204693612, - "acc_stderr": 0.012187773370741518, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.6473272490221643, + "acc_stderr": 0.012203286846053913, + "brier_score": 0.47846858672336157, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_logical_fallacies": { - "acc": 0.8895705521472392, - "acc_stderr": 0.024624937788941315, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.8957055214723927, + "acc_stderr": 0.02401351731943907, + "brier_score": 0.14625915435050918, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_high_school_us_history": { - "acc": 0.9362745098039216, - "acc_stderr": 0.01714392165552496, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.9313725490196079, + "acc_stderr": 0.017744453647073315, + "brier_score": 0.11417164363592534, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_philosophy": { - "acc": 0.8778135048231511, - "acc_stderr": 0.018600811252967916, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.8585209003215434, + "acc_stderr": 0.019794326658090552, + "brier_score": 0.21035868888892265, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_jurisprudence": { - "acc": 0.8425925925925926, - "acc_stderr": 0.03520703990517965, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.8796296296296297, + "acc_stderr": 0.031457038543062525, + "brier_score": 0.17873812246703372, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_international_law": { - "acc": 0.9338842975206612, - "acc_stderr": 0.022683403691723322, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.8842975206611571, + "acc_stderr": 0.029199802455622793, + "brier_score": 0.153677223341933, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_high_school_european_history": { - "acc": 0.8787878787878788, - "acc_stderr": 0.025485498373343223, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.8606060606060606, + "acc_stderr": 0.027045948825865366, + "brier_score": 0.21194217726890208, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_high_school_government_and_politics": { - "acc": 0.9740932642487047, - "acc_stderr": 0.011464523356953162, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.9792746113989638, + "acc_stderr": 0.01028141701190903, + "brier_score": 0.03631714843865163, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_high_school_microeconomics": { "acc": 0.9453781512605042, - "acc_stderr": 0.014760864893498496, - "timestamp": "2024-11-28T06-58-27.057743" + "acc_stderr": 0.014760864893498498, + "brier_score": 0.07367456895773429, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_high_school_geography": { - "acc": 0.9444444444444444, - "acc_stderr": 0.016319950700767395, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.9494949494949495, + "acc_stderr": 0.015602012491972257, + "brier_score": 0.09120841211492361, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_high_school_psychology": { - "acc": 0.9339449541284404, - "acc_stderr": 0.010649131487858935, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.9357798165137615, + "acc_stderr": 0.010510494713201436, + "brier_score": 0.10160597088346875, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_public_relations": { - "acc": 0.7636363636363637, - "acc_stderr": 0.04069306319721377, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.7181818181818181, + "acc_stderr": 0.043091187099464585, + "brier_score": 0.35377944942169043, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_us_foreign_policy": { - "acc": 0.96, - "acc_stderr": 0.019694638556693223, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.97, + "acc_stderr": 0.017144660799776522, + "brier_score": 0.0683088491484518, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_sociology": { "acc": 0.9104477611940298, "acc_stderr": 0.02019067053502794, - "timestamp": "2024-11-28T06-58-27.057743" + "brier_score": 0.12880498390957273, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_high_school_macroeconomics": { - "acc": 0.8769230769230769, - "acc_stderr": 0.016656903282068206, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.882051282051282, + "acc_stderr": 0.016353801778303412, + "brier_score": 0.16350288975640653, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_security_studies": { - "acc": 0.8489795918367347, - "acc_stderr": 0.022923004094736858, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.8244897959183674, + "acc_stderr": 0.024352800722970015, + "brier_score": 0.24128587783336555, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_professional_psychology": { - "acc": 0.8709150326797386, - "acc_stderr": 0.01356454163404412, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.8725490196078431, + "acc_stderr": 0.013491054682536602, + "brier_score": 0.20267167261212077, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_human_sexuality": { - "acc": 0.9236641221374046, - "acc_stderr": 0.023288939536173784, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.9083969465648855, + "acc_stderr": 0.025300035578642962, + "brier_score": 0.13988432928671568, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_econometrics": { - "acc": 0.7807017543859649, - "acc_stderr": 0.038924311065187546, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.7543859649122807, + "acc_stderr": 0.040493392977481425, + "brier_score": 0.3434665275290386, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_miscellaneous": { - "acc": 0.9195402298850575, - "acc_stderr": 0.009726831316141838, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.9157088122605364, + "acc_stderr": 0.009934966499513812, + "brier_score": 0.11813100395381679, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_marketing": { - "acc": 0.9145299145299145, - "acc_stderr": 0.018315891685625838, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.9615384615384616, + "acc_stderr": 0.012598495809238823, + "brier_score": 0.0743330499000944, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_management": { - "acc": 0.883495145631068, - "acc_stderr": 0.03176683948640407, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.8543689320388349, + "acc_stderr": 0.0349260647662379, + "brier_score": 0.19825790177318575, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_nutrition": { - "acc": 0.869281045751634, - "acc_stderr": 0.01930187362421529, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.8856209150326797, + "acc_stderr": 0.018224151682733437, + "brier_score": 0.16557819109709535, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_medical_genetics": { "acc": 0.91, - "acc_stderr": 0.028762349126466153, - "timestamp": "2024-11-28T06-58-27.057743" + "acc_stderr": 0.028762349126466115, + "brier_score": 0.13088017617534822, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_human_aging": { - "acc": 0.8161434977578476, - "acc_stderr": 0.02599837909235651, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.8026905829596412, + "acc_stderr": 0.02670985334496796, + "brier_score": 0.2765866371639233, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_professional_medicine": { - "acc": 0.8970588235294118, - "acc_stderr": 0.01845951895538869, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.8713235294117647, + "acc_stderr": 0.02034017315389899, + "brier_score": 0.1764664227247683, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_college_medicine": { - "acc": 0.8265895953757225, - "acc_stderr": 0.028868107874970635, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.7976878612716763, + "acc_stderr": 0.030631145539198823, + "brier_score": 0.2737864492370116, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_business_ethics": { - "acc": 0.82, - "acc_stderr": 0.03861229196653693, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.79, + "acc_stderr": 0.040936018074033256, + "brier_score": 0.21591269527179754, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_clinical_knowledge": { - "acc": 0.8754716981132076, - "acc_stderr": 0.020321376630696202, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.8641509433962264, + "acc_stderr": 0.021087308622439866, + "brier_score": 0.17097931864162125, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_global_facts": { - "acc": 0.71, - "acc_stderr": 0.045604802157206845, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.65, + "acc_stderr": 0.04793724854411018, + "brier_score": 0.467281396851418, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_virology": { - "acc": 0.572289156626506, - "acc_stderr": 0.038515976837185335, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.5783132530120482, + "acc_stderr": 0.03844453181770917, + "brier_score": 0.6918072761365407, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_professional_accounting": { - "acc": 0.6843971631205674, - "acc_stderr": 0.027724989449509314, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.6560283687943262, + "acc_stderr": 0.02833801742861132, + "brier_score": 0.4293875955658281, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_college_physics": { - "acc": 0.7450980392156863, - "acc_stderr": 0.0433643270799318, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.7156862745098039, + "acc_stderr": 0.04488482852329017, + "brier_score": 0.35484228528248163, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_high_school_physics": { - "acc": 0.7615894039735099, - "acc_stderr": 0.0347918557259966, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.7284768211920529, + "acc_stderr": 0.03631329803969653, + "brier_score": 0.3364897926867841, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_high_school_biology": { - "acc": 0.9516129032258065, - "acc_stderr": 0.012207189992293643, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.9451612903225807, + "acc_stderr": 0.012951418509899204, + "brier_score": 0.09968722250890918, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_college_biology": { - "acc": 0.9375, - "acc_stderr": 0.02024219611347799, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.9444444444444444, + "acc_stderr": 0.01915507853243362, + "brier_score": 0.10432433862178721, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_anatomy": { - "acc": 0.7851851851851852, - "acc_stderr": 0.03547854198560824, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.7925925925925926, + "acc_stderr": 0.035025531706783165, + "brier_score": 0.28673430831966384, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_college_chemistry": { - "acc": 0.62, - "acc_stderr": 0.04878317312145632, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.64, + "acc_stderr": 0.04824181513244218, + "brier_score": 0.46658750491890466, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_computer_security": { - "acc": 0.86, - "acc_stderr": 0.03487350880197771, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.87, + "acc_stderr": 0.03379976689896309, + "brier_score": 0.20797563626348528, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_college_computer_science": { - "acc": 0.79, - "acc_stderr": 0.040936018074033256, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.78, + "acc_stderr": 0.04163331998932263, + "brier_score": 0.2959779141593885, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_astronomy": { - "acc": 0.9407894736842105, - "acc_stderr": 0.019206897196800306, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.9210526315789473, + "acc_stderr": 0.021944342818247954, + "brier_score": 0.1330036230669275, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_college_mathematics": { - "acc": 0.74, - "acc_stderr": 0.044084400227680794, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.65, + "acc_stderr": 0.0479372485441102, + "brier_score": 0.46383485729525886, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_conceptual_physics": { - "acc": 0.902127659574468, - "acc_stderr": 0.019424777705573392, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.8638297872340426, + "acc_stderr": 0.022420599304382047, + "brier_score": 0.17357236053215522, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_abstract_algebra": { - "acc": 0.72, - "acc_stderr": 0.04512608598542127, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.63, + "acc_stderr": 0.04852365870939099, + "brier_score": 0.4681167799528041, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_high_school_computer_science": { - "acc": 0.94, - "acc_stderr": 0.02386832565759419, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.91, + "acc_stderr": 0.02876234912646612, + "brier_score": 0.12013805217460581, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_machine_learning": { - "acc": 0.7946428571428571, - "acc_stderr": 0.03834241021419073, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.7232142857142857, + "acc_stderr": 0.042466243366976256, + "brier_score": 0.36391516376263827, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_high_school_chemistry": { - "acc": 0.7684729064039408, - "acc_stderr": 0.029678333141444444, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.7832512315270936, + "acc_stderr": 0.028990331252516235, + "brier_score": 0.3013394560585245, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_high_school_statistics": { "acc": 0.8518518518518519, - "acc_stderr": 0.02422762927372837, - "timestamp": "2024-11-28T06-58-27.057743" + "acc_stderr": 0.024227629273728356, + "brier_score": 0.22193795598289576, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_elementary_mathematics": { - "acc": 0.9021164021164021, - "acc_stderr": 0.015304374225091419, - "timestamp": "2024-11-28T06-58-27.057743" + "acc": 0.8941798941798942, + "acc_stderr": 0.015842578614188925, + "brier_score": 0.16314711146382785, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_electrical_engineering": { "acc": 0.8206896551724138, "acc_stderr": 0.03196766433373187, - "timestamp": "2024-11-28T06-58-27.057743" + "brier_score": 0.29135716126824623, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "mmlu_high_school_mathematics": { "acc": 0.6518518518518519, "acc_stderr": 0.029045600290616255, - "timestamp": "2024-11-28T06-58-27.057743" + "brier_score": 0.42336015705679986, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-56-04.985961" }, "arc_challenge": { "acc": 0.6621160409556314, diff --git a/metadata/model_scores/results_Qwen__Qwen2.5-3B.json b/metadata/model_scores/results_Qwen__Qwen2.5-3B.json index def43cc..991ea53 100644 --- a/metadata/model_scores/results_Qwen__Qwen2.5-3B.json +++ b/metadata/model_scores/results_Qwen__Qwen2.5-3B.json @@ -1,292 +1,406 @@ { "model_name": "Qwen__Qwen2.5-3B", - "last_updated": "2024-12-04 11:22:42.423236", + "last_updated": "2024-12-19 13:38:01.135264", "results": { "harness": { "mmlu_world_religions": { - "acc": 0.8304093567251462, - "acc_stderr": 0.02878210810540171, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.8187134502923976, + "acc_stderr": 0.029547741687640038, + "brier_score": 0.24563834735905882, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_formal_logic": { - "acc": 0.46825396825396826, - "acc_stderr": 0.04463112720677173, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.49206349206349204, + "acc_stderr": 0.044715725362943486, + "brier_score": 0.6296355052360546, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_prehistory": { - "acc": 0.75, - "acc_stderr": 0.02409347123262133, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.7253086419753086, + "acc_stderr": 0.024836057868294677, + "brier_score": 0.3801587127112863, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_moral_scenarios": { - "acc": 0.2636871508379888, - "acc_stderr": 0.01473692638376197, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.2435754189944134, + "acc_stderr": 0.01435591196476786, + "brier_score": 0.962622291102738, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_high_school_world_history": { - "acc": 0.8438818565400844, - "acc_stderr": 0.02362715946031869, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.8227848101265823, + "acc_stderr": 0.02485636418450323, + "brier_score": 0.24574863982281853, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_moral_disputes": { - "acc": 0.7196531791907514, - "acc_stderr": 0.024182427496577615, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.6763005780346821, + "acc_stderr": 0.025190181327608408, + "brier_score": 0.4249982858165409, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_professional_law": { - "acc": 0.4791395045632334, - "acc_stderr": 0.012759117066518019, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.4869621903520209, + "acc_stderr": 0.012765893883835332, + "brier_score": 0.6361156940456084, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_logical_fallacies": { - "acc": 0.7791411042944786, - "acc_stderr": 0.03259177392742178, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.7914110429447853, + "acc_stderr": 0.03192193448934725, + "brier_score": 0.29033540862233154, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_high_school_us_history": { - "acc": 0.8431372549019608, - "acc_stderr": 0.02552472232455335, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.8333333333333334, + "acc_stderr": 0.026156867523931055, + "brier_score": 0.2500351675424673, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_philosophy": { - "acc": 0.7170418006430869, - "acc_stderr": 0.025583062489984834, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.7138263665594855, + "acc_stderr": 0.025670259242188947, + "brier_score": 0.3933299118993118, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_jurisprudence": { - "acc": 0.7685185185185185, - "acc_stderr": 0.04077494709252626, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.8148148148148148, + "acc_stderr": 0.03755265865037181, + "brier_score": 0.27522692177544283, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_international_law": { - "acc": 0.768595041322314, - "acc_stderr": 0.03849856098794088, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.8264462809917356, + "acc_stderr": 0.03457272836917669, + "brier_score": 0.27296504783771897, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_high_school_european_history": { - "acc": 0.7757575757575758, - "acc_stderr": 0.032568666616811015, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.7636363636363637, + "acc_stderr": 0.033175059300091805, + "brier_score": 0.33472155884729393, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_high_school_government_and_politics": { - "acc": 0.8549222797927462, - "acc_stderr": 0.025416343096306422, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.8601036269430051, + "acc_stderr": 0.025033870583015178, + "brier_score": 0.19549088843628196, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_high_school_microeconomics": { - "acc": 0.7815126050420168, - "acc_stderr": 0.02684151432295893, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.8067226890756303, + "acc_stderr": 0.025649470265889193, + "brier_score": 0.3036142514551187, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_high_school_geography": { - "acc": 0.8080808080808081, - "acc_stderr": 0.02805779167298901, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.7929292929292929, + "acc_stderr": 0.028869778460267066, + "brier_score": 0.28880779032422577, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_high_school_psychology": { - "acc": 0.8587155963302753, - "acc_stderr": 0.01493386898702808, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.8477064220183487, + "acc_stderr": 0.015405084393157069, + "brier_score": 0.20033463599748524, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_public_relations": { - "acc": 0.7181818181818181, - "acc_stderr": 0.04309118709946458, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.7, + "acc_stderr": 0.04389311454644287, + "brier_score": 0.4009798205516975, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_us_foreign_policy": { - "acc": 0.84, - "acc_stderr": 0.036845294917747094, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.83, + "acc_stderr": 0.0377525168068637, + "brier_score": 0.23176869938521993, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_sociology": { - "acc": 0.835820895522388, - "acc_stderr": 0.026193923544454115, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.8258706467661692, + "acc_stderr": 0.026814951200421603, + "brier_score": 0.2593726027414835, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_high_school_macroeconomics": { - "acc": 0.7076923076923077, - "acc_stderr": 0.02306043838085775, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.6820512820512821, + "acc_stderr": 0.023610884308927865, + "brier_score": 0.39070268561227034, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_security_studies": { - "acc": 0.7346938775510204, - "acc_stderr": 0.028263889943784603, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.7306122448979592, + "acc_stderr": 0.02840125202902294, + "brier_score": 0.3572075223141421, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_professional_psychology": { - "acc": 0.6977124183006536, - "acc_stderr": 0.018579232711113895, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.7124183006535948, + "acc_stderr": 0.018311653053648222, + "brier_score": 0.40299102758795385, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_human_sexuality": { - "acc": 0.7709923664122137, - "acc_stderr": 0.036853466317118506, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.7480916030534351, + "acc_stderr": 0.038073871163060866, + "brier_score": 0.33656144859824927, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_econometrics": { - "acc": 0.5614035087719298, - "acc_stderr": 0.04668000738510455, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.5263157894736842, + "acc_stderr": 0.046970851366478626, + "brier_score": 0.6314189534004948, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_miscellaneous": { - "acc": 0.8007662835249042, - "acc_stderr": 0.014283378044296413, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.7994891443167306, + "acc_stderr": 0.014317653708594209, + "brier_score": 0.2675858404124839, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_marketing": { - "acc": 0.8974358974358975, - "acc_stderr": 0.019875655027867447, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.8846153846153846, + "acc_stderr": 0.02093019318517933, + "brier_score": 0.17685456520590553, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_management": { - "acc": 0.7961165048543689, - "acc_stderr": 0.03989139859531772, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.8155339805825242, + "acc_stderr": 0.03840423627288276, + "brier_score": 0.22934336958888674, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_nutrition": { - "acc": 0.7516339869281046, - "acc_stderr": 0.024739981355113592, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.761437908496732, + "acc_stderr": 0.024404394928087866, + "brier_score": 0.3327031238651498, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_medical_genetics": { - "acc": 0.8, - "acc_stderr": 0.04020151261036845, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.73, + "acc_stderr": 0.04461960433384741, + "brier_score": 0.3168223886209323, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_human_aging": { - "acc": 0.7040358744394619, - "acc_stderr": 0.030636591348699813, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.7399103139013453, + "acc_stderr": 0.029442495585857473, + "brier_score": 0.4084280177407385, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_professional_medicine": { "acc": 0.6580882352941176, "acc_stderr": 0.02881472242225418, - "timestamp": "2024-11-22T21-40-10.451010" + "brier_score": 0.47004532244711683, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_college_medicine": { - "acc": 0.6820809248554913, - "acc_stderr": 0.035506839891655796, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.6589595375722543, + "acc_stderr": 0.036146654241808254, + "brier_score": 0.4127036479388521, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_business_ethics": { - "acc": 0.71, - "acc_stderr": 0.045604802157206845, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.72, + "acc_stderr": 0.04512608598542129, + "brier_score": 0.3309723529677748, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_clinical_knowledge": { - "acc": 0.7245283018867924, - "acc_stderr": 0.027495663683724046, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.7320754716981132, + "acc_stderr": 0.027257260322494845, + "brier_score": 0.3467972625679778, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_global_facts": { - "acc": 0.39, - "acc_stderr": 0.04902071300001974, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.36, + "acc_stderr": 0.04824181513244218, + "brier_score": 0.7108003167393311, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_virology": { - "acc": 0.4819277108433735, - "acc_stderr": 0.038899512528272166, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.5060240963855421, + "acc_stderr": 0.03892212195333045, + "brier_score": 0.7280065109273901, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_professional_accounting": { - "acc": 0.4929078014184397, - "acc_stderr": 0.02982449855912901, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.5141843971631206, + "acc_stderr": 0.02981549448368206, + "brier_score": 0.5810576259817505, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_college_physics": { - "acc": 0.49019607843137253, + "acc": 0.5098039215686274, "acc_stderr": 0.04974229460422817, - "timestamp": "2024-11-22T21-40-10.451010" + "brier_score": 0.5644665416176843, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_high_school_physics": { - "acc": 0.45695364238410596, - "acc_stderr": 0.04067325174247443, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.41721854304635764, + "acc_stderr": 0.040261414976346104, + "brier_score": 0.6875609768063202, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_high_school_biology": { - "acc": 0.8225806451612904, - "acc_stderr": 0.021732540689329272, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.8096774193548387, + "acc_stderr": 0.022331707611823067, + "brier_score": 0.26083194119588576, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_college_biology": { - "acc": 0.7638888888888888, - "acc_stderr": 0.03551446610810826, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.7013888888888888, + "acc_stderr": 0.03827052357950756, + "brier_score": 0.33389341524347393, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_anatomy": { - "acc": 0.6296296296296297, - "acc_stderr": 0.04171654161354544, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.6222222222222222, + "acc_stderr": 0.04188307537595853, + "brier_score": 0.48947933132471644, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_college_chemistry": { - "acc": 0.48, - "acc_stderr": 0.050211673156867795, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.49, + "acc_stderr": 0.05024183937956913, + "brier_score": 0.6023101953688079, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_computer_security": { - "acc": 0.8, - "acc_stderr": 0.04020151261036846, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.77, + "acc_stderr": 0.04229525846816505, + "brier_score": 0.3091005241781668, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_college_computer_science": { - "acc": 0.6, - "acc_stderr": 0.04923659639173309, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.58, + "acc_stderr": 0.049604496374885836, + "brier_score": 0.5087894407032691, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_astronomy": { - "acc": 0.743421052631579, - "acc_stderr": 0.035541803680256896, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.6842105263157895, + "acc_stderr": 0.037827289808654685, + "brier_score": 0.3960447226399447, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_college_mathematics": { - "acc": 0.44, - "acc_stderr": 0.04988876515698589, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.4, + "acc_stderr": 0.049236596391733084, + "brier_score": 0.6804165912352971, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_conceptual_physics": { - "acc": 0.6425531914893617, - "acc_stderr": 0.031329417894764254, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.676595744680851, + "acc_stderr": 0.030579442773610344, + "brier_score": 0.4625355830633662, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_abstract_algebra": { - "acc": 0.38, - "acc_stderr": 0.048783173121456316, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.5, + "acc_stderr": 0.050251890762960605, + "brier_score": 0.6589979467745916, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_high_school_computer_science": { - "acc": 0.76, - "acc_stderr": 0.042923469599092816, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.74, + "acc_stderr": 0.04408440022768078, + "brier_score": 0.3271117286082809, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_machine_learning": { - "acc": 0.4642857142857143, - "acc_stderr": 0.04733667890053756, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.45535714285714285, + "acc_stderr": 0.04726835553719099, + "brier_score": 0.6080303321956573, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_high_school_chemistry": { - "acc": 0.6059113300492611, - "acc_stderr": 0.03438157967036545, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.5862068965517241, + "acc_stderr": 0.03465304488406796, + "brier_score": 0.5089021864446602, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_high_school_statistics": { - "acc": 0.625, - "acc_stderr": 0.033016908987210894, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.6111111111111112, + "acc_stderr": 0.033247089118091176, + "brier_score": 0.5083961832308744, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_elementary_mathematics": { - "acc": 0.5978835978835979, - "acc_stderr": 0.02525303255499769, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.5740740740740741, + "acc_stderr": 0.025467149045469543, + "brier_score": 0.4992870946014208, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_electrical_engineering": { - "acc": 0.7034482758620689, - "acc_stderr": 0.03806142687309992, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.6413793103448275, + "acc_stderr": 0.03996629574876718, + "brier_score": 0.46942897274818185, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "mmlu_high_school_mathematics": { - "acc": 0.5, - "acc_stderr": 0.030485538042484616, - "timestamp": "2024-11-22T21-40-10.451010" + "acc": 0.5185185185185185, + "acc_stderr": 0.03046462171889531, + "brier_score": 0.6094137662078477, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-00-32.590210" }, "arc_challenge": { "acc": 0.5332764505119454, diff --git a/metadata/model_scores/results_Salesforce__codegen-16B-multi.json b/metadata/model_scores/results_Salesforce__codegen-16B-multi.json index 7c95152..3bf456b 100644 --- a/metadata/model_scores/results_Salesforce__codegen-16B-multi.json +++ b/metadata/model_scores/results_Salesforce__codegen-16B-multi.json @@ -1,6 +1,6 @@ { "model_name": "Salesforce/codegen-16B-multi", - "last_updated": "2024-12-04 11:24:21.091486", + "last_updated": "2024-12-19 13:40:12.946693", "results": { "harness": { "minerva_math_precalc": { @@ -228,289 +228,403 @@ "timestamp": "2024-06-14T04-05-35.450863" }, "mmlu_world_religions": { - "acc": 0.32748538011695905, - "acc_stderr": 0.035993357714560276, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.3216374269005848, + "acc_stderr": 0.03582529442573122, + "brier_score": 0.7627873242855722, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_formal_logic": { - "acc": 0.30158730158730157, - "acc_stderr": 0.04104947269903394, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.25396825396825395, + "acc_stderr": 0.03893259610604673, + "brier_score": 0.7689830422591174, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_prehistory": { - "acc": 0.30864197530864196, - "acc_stderr": 0.025702640260603756, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.25, + "acc_stderr": 0.02409347123262133, + "brier_score": 0.7807142929224355, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_moral_scenarios": { - "acc": 0.24692737430167597, - "acc_stderr": 0.014422292204808864, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.24022346368715083, + "acc_stderr": 0.014288343803925293, + "brier_score": 0.7845479297986755, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_high_school_world_history": { - "acc": 0.270042194092827, - "acc_stderr": 0.028900721906293426, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.2742616033755274, + "acc_stderr": 0.029041333510598046, + "brier_score": 0.7690532333296719, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_moral_disputes": { - "acc": 0.2254335260115607, - "acc_stderr": 0.022497230190967547, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.30057803468208094, + "acc_stderr": 0.024685316867257792, + "brier_score": 0.7658024042802853, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_professional_law": { - "acc": 0.2588005215123859, - "acc_stderr": 0.011186109046564613, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.2470664928292047, + "acc_stderr": 0.011015752255279319, + "brier_score": 0.7741547742598577, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_logical_fallacies": { - "acc": 0.2331288343558282, - "acc_stderr": 0.033220157957767414, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.32515337423312884, + "acc_stderr": 0.03680350371286461, + "brier_score": 0.7559119563737517, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_high_school_us_history": { - "acc": 0.27450980392156865, - "acc_stderr": 0.031321798030832904, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.28431372549019607, + "acc_stderr": 0.03166009679399813, + "brier_score": 0.7646895380414529, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_philosophy": { - "acc": 0.2733118971061093, - "acc_stderr": 0.025311765975426115, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.2282958199356913, + "acc_stderr": 0.023839303311398222, + "brier_score": 0.7927169636500151, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_jurisprudence": { "acc": 0.19444444444444445, - "acc_stderr": 0.03826076324884863, - "timestamp": "2024-11-20T10-20-44.927267" + "acc_stderr": 0.03826076324884864, + "brier_score": 0.8074905485544689, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_international_law": { - "acc": 0.2727272727272727, - "acc_stderr": 0.04065578140908705, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.3305785123966942, + "acc_stderr": 0.04294340845212094, + "brier_score": 0.7338956031867301, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_high_school_european_history": { - "acc": 0.2909090909090909, - "acc_stderr": 0.03546563019624336, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.24848484848484848, + "acc_stderr": 0.03374402644139404, + "brier_score": 0.7777113191258419, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_high_school_government_and_politics": { "acc": 0.25906735751295334, - "acc_stderr": 0.031618779179354135, - "timestamp": "2024-11-20T10-20-44.927267" + "acc_stderr": 0.03161877917935411, + "brier_score": 0.7777099996496186, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_high_school_microeconomics": { - "acc": 0.29411764705882354, - "acc_stderr": 0.02959732973097809, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.2857142857142857, + "acc_stderr": 0.029344572500634332, + "brier_score": 0.771347524950132, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_high_school_geography": { - "acc": 0.24242424242424243, - "acc_stderr": 0.030532892233932026, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.29292929292929293, + "acc_stderr": 0.032424979581788166, + "brier_score": 0.7628813204354998, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_high_school_psychology": { - "acc": 0.21834862385321102, - "acc_stderr": 0.017712600528722717, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.25688073394495414, + "acc_stderr": 0.018732492928342472, + "brier_score": 0.7765931500840567, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_public_relations": { - "acc": 0.21818181818181817, - "acc_stderr": 0.03955932861795833, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.24545454545454545, + "acc_stderr": 0.04122066502878284, + "brier_score": 0.8097039663378572, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_us_foreign_policy": { "acc": 0.33, - "acc_stderr": 0.04725815626252604, - "timestamp": "2024-11-20T10-20-44.927267" + "acc_stderr": 0.04725815626252605, + "brier_score": 0.7566402696284292, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_sociology": { "acc": 0.2537313432835821, "acc_stderr": 0.030769444967296024, - "timestamp": "2024-11-20T10-20-44.927267" + "brier_score": 0.7699091524176438, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_high_school_macroeconomics": { - "acc": 0.26153846153846155, - "acc_stderr": 0.022282141204204423, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.2205128205128205, + "acc_stderr": 0.02102067268082791, + "brier_score": 0.7887454152198857, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_security_studies": { - "acc": 0.3795918367346939, - "acc_stderr": 0.031067211262872478, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.34285714285714286, + "acc_stderr": 0.030387262919547735, + "brier_score": 0.7482473159250772, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_professional_psychology": { - "acc": 0.25, - "acc_stderr": 0.01751781884501444, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.29411764705882354, + "acc_stderr": 0.01843342764940189, + "brier_score": 0.767820703187618, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_human_sexuality": { - "acc": 0.33587786259541985, - "acc_stderr": 0.04142313771996665, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.25190839694656486, + "acc_stderr": 0.03807387116306085, + "brier_score": 0.7693791681119263, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_econometrics": { - "acc": 0.23684210526315788, - "acc_stderr": 0.039994238792813365, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.16666666666666666, + "acc_stderr": 0.03505859682597264, + "brier_score": 0.8387501182260378, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_miscellaneous": { - "acc": 0.2669220945083014, - "acc_stderr": 0.01581845089477755, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.25925925925925924, + "acc_stderr": 0.01567100600933957, + "brier_score": 0.7907263754130816, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_marketing": { - "acc": 0.27350427350427353, - "acc_stderr": 0.029202540153431177, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.24786324786324787, + "acc_stderr": 0.028286324075564407, + "brier_score": 0.7746616650029644, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_management": { - "acc": 0.1941747572815534, - "acc_stderr": 0.03916667762822585, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.1650485436893204, + "acc_stderr": 0.03675668832233188, + "brier_score": 0.8462996513443711, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_nutrition": { - "acc": 0.25163398692810457, - "acc_stderr": 0.024848018263875192, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.2549019607843137, + "acc_stderr": 0.02495418432487991, + "brier_score": 0.7801372618222232, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_medical_genetics": { - "acc": 0.35, - "acc_stderr": 0.04793724854411018, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.26, + "acc_stderr": 0.04408440022768079, + "brier_score": 0.7627202267658829, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_human_aging": { - "acc": 0.3183856502242152, - "acc_stderr": 0.03126580522513713, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.23766816143497757, + "acc_stderr": 0.028568079464714277, + "brier_score": 0.7865910674078115, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_professional_medicine": { - "acc": 0.2610294117647059, - "acc_stderr": 0.02667925227010312, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.31985294117647056, + "acc_stderr": 0.028332959514031218, + "brier_score": 0.7558514526049404, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_college_medicine": { - "acc": 0.2254335260115607, - "acc_stderr": 0.03186209851641143, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.19653179190751446, + "acc_stderr": 0.030299574664788137, + "brier_score": 0.794320488478051, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_business_ethics": { - "acc": 0.31, - "acc_stderr": 0.04648231987117316, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.26, + "acc_stderr": 0.04408440022768078, + "brier_score": 0.7641813854135013, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_clinical_knowledge": { - "acc": 0.24528301886792453, - "acc_stderr": 0.02648035717989571, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.2490566037735849, + "acc_stderr": 0.026616482980501708, + "brier_score": 0.7981102158252912, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_global_facts": { - "acc": 0.18, - "acc_stderr": 0.03861229196653694, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.35, + "acc_stderr": 0.0479372485441102, + "brier_score": 0.7791554852331415, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_virology": { - "acc": 0.26506024096385544, - "acc_stderr": 0.03436024037944967, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.23493975903614459, + "acc_stderr": 0.03300533186128922, + "brier_score": 0.7982656112392703, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_professional_accounting": { - "acc": 0.1879432624113475, - "acc_stderr": 0.023305230769714243, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.22340425531914893, + "acc_stderr": 0.024847921358063962, + "brier_score": 0.8332174045787025, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_college_physics": { - "acc": 0.22549019607843138, - "acc_stderr": 0.041583075330832865, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.12745098039215685, + "acc_stderr": 0.03318224921942076, + "brier_score": 0.8445045042279045, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_high_school_physics": { - "acc": 0.33774834437086093, - "acc_stderr": 0.0386155754625517, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.2781456953642384, + "acc_stderr": 0.03658603262763743, + "brier_score": 0.7806254894541519, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_high_school_biology": { - "acc": 0.3032258064516129, - "acc_stderr": 0.026148685930671753, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.26129032258064516, + "acc_stderr": 0.024993053397764815, + "brier_score": 0.77598402053733, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_college_biology": { "acc": 0.3055555555555556, "acc_stderr": 0.03852084696008534, - "timestamp": "2024-11-20T10-20-44.927267" + "brier_score": 0.7581486611163988, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_anatomy": { - "acc": 0.28888888888888886, - "acc_stderr": 0.0391545063041425, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.31851851851851853, + "acc_stderr": 0.0402477840197711, + "brier_score": 0.7632834128768158, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_college_chemistry": { - "acc": 0.39, - "acc_stderr": 0.04902071300001975, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.19, + "acc_stderr": 0.03942772444036623, + "brier_score": 0.8187668700393952, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_computer_security": { - "acc": 0.33, - "acc_stderr": 0.04725815626252605, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "brier_score": 0.7529534087015408, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_college_computer_science": { - "acc": 0.24, - "acc_stderr": 0.04292346959909283, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.27, + "acc_stderr": 0.0446196043338474, + "brier_score": 0.7637992365474011, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_astronomy": { - "acc": 0.27631578947368424, - "acc_stderr": 0.03639057569952925, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.2631578947368421, + "acc_stderr": 0.03583496176361063, + "brier_score": 0.7669964212763819, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_college_mathematics": { - "acc": 0.32, - "acc_stderr": 0.046882617226215034, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.24, + "acc_stderr": 0.042923469599092816, + "brier_score": 0.844011071165082, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_conceptual_physics": { - "acc": 0.2680851063829787, - "acc_stderr": 0.028957342788342347, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.26382978723404255, + "acc_stderr": 0.028809989854102973, + "brier_score": 0.7723433608888436, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_abstract_algebra": { - "acc": 0.23, - "acc_stderr": 0.04229525846816505, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 0.9370967381574306, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_high_school_computer_science": { - "acc": 0.3, - "acc_stderr": 0.046056618647183814, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.37, + "acc_stderr": 0.04852365870939099, + "brier_score": 0.7451369908253416, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_machine_learning": { - "acc": 0.2767857142857143, - "acc_stderr": 0.04246624336697626, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.30357142857142855, + "acc_stderr": 0.04364226155841044, + "brier_score": 0.8142410358074835, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_high_school_chemistry": { - "acc": 0.2512315270935961, - "acc_stderr": 0.030516530732694436, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.21182266009852216, + "acc_stderr": 0.028748983689941072, + "brier_score": 0.8128684448501712, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_high_school_statistics": { - "acc": 0.375, - "acc_stderr": 0.033016908987210894, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.1712962962962963, + "acc_stderr": 0.025695341643824685, + "brier_score": 0.8388018598400845, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_elementary_mathematics": { - "acc": 0.26455026455026454, - "acc_stderr": 0.02271746789770862, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.2698412698412698, + "acc_stderr": 0.022860838309232072, + "brier_score": 0.8393886511286482, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_electrical_engineering": { - "acc": 0.3310344827586207, - "acc_stderr": 0.039215453124671215, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.3448275862068966, + "acc_stderr": 0.03960933549451207, + "brier_score": 0.7445540897601858, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "mmlu_high_school_mathematics": { - "acc": 0.26296296296296295, - "acc_stderr": 0.026842057873833706, - "timestamp": "2024-11-20T10-20-44.927267" + "acc": 0.2518518518518518, + "acc_stderr": 0.02646611753895991, + "brier_score": 0.8483657210198746, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-42-25.772351" }, "arc_challenge": { "acc": 0.28498293515358364, diff --git a/metadata/model_scores/results_Salesforce__codegen-2B-multi.json b/metadata/model_scores/results_Salesforce__codegen-2B-multi.json index 67aceec..909e8eb 100644 --- a/metadata/model_scores/results_Salesforce__codegen-2B-multi.json +++ b/metadata/model_scores/results_Salesforce__codegen-2B-multi.json @@ -1,6 +1,6 @@ { "model_name": "Salesforce/codegen-2B-multi", - "last_updated": "2024-12-04 11:24:46.619250", + "last_updated": "2024-12-19 13:40:42.041483", "results": { "harness": { "minerva_math_precalc": { @@ -228,289 +228,403 @@ "timestamp": "2024-06-13T20-35-34.571703" }, "mmlu_world_religions": { - "acc": 0.23391812865497075, - "acc_stderr": 0.03246721765117826, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.3216374269005848, + "acc_stderr": 0.03582529442573122, + "brier_score": 0.7546803865019402, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_formal_logic": { - "acc": 0.12698412698412698, - "acc_stderr": 0.029780417522688434, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.23809523809523808, + "acc_stderr": 0.03809523809523812, + "brier_score": 0.7861060931204427, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_prehistory": { "acc": 0.23148148148148148, "acc_stderr": 0.023468429832451166, - "timestamp": "2024-11-11T08-57-26.989242" + "brier_score": 0.7956158233024767, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_moral_scenarios": { - "acc": 0.2424581005586592, - "acc_stderr": 0.014333522059217887, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.23798882681564246, + "acc_stderr": 0.014242630070574885, + "brier_score": 0.7758538855796485, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_high_school_world_history": { - "acc": 0.2109704641350211, - "acc_stderr": 0.02655837250266192, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.23628691983122363, + "acc_stderr": 0.027652153144159267, + "brier_score": 0.7881333404455585, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_moral_disputes": { - "acc": 0.2543352601156069, - "acc_stderr": 0.02344582627654555, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.27167630057803466, + "acc_stderr": 0.02394851290546835, + "brier_score": 0.770457570332024, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_professional_law": { - "acc": 0.24902216427640156, - "acc_stderr": 0.01104489226404077, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.24119947848761408, + "acc_stderr": 0.010926496102034952, + "brier_score": 0.7875492624430832, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_logical_fallacies": { - "acc": 0.26993865030674846, - "acc_stderr": 0.03487825168497892, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.27607361963190186, + "acc_stderr": 0.0351238528370505, + "brier_score": 0.7756305991459267, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_high_school_us_history": { - "acc": 0.24509803921568626, - "acc_stderr": 0.030190282453501954, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.2549019607843137, + "acc_stderr": 0.030587591351604233, + "brier_score": 0.782571487009046, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_philosophy": { - "acc": 0.3022508038585209, - "acc_stderr": 0.02608270069539966, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.21543408360128619, + "acc_stderr": 0.02335022547547143, + "brier_score": 0.7930833367764941, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_jurisprudence": { - "acc": 0.25925925925925924, - "acc_stderr": 0.04236511258094635, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.2222222222222222, + "acc_stderr": 0.040191074725573483, + "brier_score": 0.7829344122884471, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_international_law": { - "acc": 0.35537190082644626, - "acc_stderr": 0.0436923632657398, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.30578512396694213, + "acc_stderr": 0.04205953933884122, + "brier_score": 0.7516569535072102, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_high_school_european_history": { - "acc": 0.23030303030303031, - "acc_stderr": 0.032876667586034886, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.20606060606060606, + "acc_stderr": 0.03158415324047709, + "brier_score": 0.8014641560807866, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_high_school_government_and_politics": { - "acc": 0.34196891191709844, - "acc_stderr": 0.03423465100104284, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.23834196891191708, + "acc_stderr": 0.03074890536390988, + "brier_score": 0.8255799819870075, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_high_school_microeconomics": { - "acc": 0.21008403361344538, - "acc_stderr": 0.026461398717471874, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.18487394957983194, + "acc_stderr": 0.025215992877954202, + "brier_score": 0.8322855849349483, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_high_school_geography": { - "acc": 0.23232323232323232, - "acc_stderr": 0.030088629490217487, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.22727272727272727, + "acc_stderr": 0.02985751567338641, + "brier_score": 0.8086400814182112, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_high_school_psychology": { - "acc": 0.21467889908256882, - "acc_stderr": 0.01760430414925649, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.22201834862385322, + "acc_stderr": 0.01781884956479663, + "brier_score": 0.8065734496672069, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_public_relations": { - "acc": 0.22727272727272727, - "acc_stderr": 0.04013964554072774, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.2545454545454545, + "acc_stderr": 0.04172343038705383, + "brier_score": 0.7773867180018731, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_us_foreign_policy": { - "acc": 0.29, - "acc_stderr": 0.045604802157206845, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "brier_score": 0.7970292919788169, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_sociology": { - "acc": 0.22885572139303484, - "acc_stderr": 0.02970528405677245, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.24378109452736318, + "acc_stderr": 0.030360490154014652, + "brier_score": 0.8042877580813117, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_high_school_macroeconomics": { - "acc": 0.26666666666666666, - "acc_stderr": 0.022421273612923707, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.19743589743589743, + "acc_stderr": 0.020182646968674844, + "brier_score": 0.8294006145780994, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_security_studies": { - "acc": 0.24081632653061225, - "acc_stderr": 0.027372942201788163, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.17142857142857143, + "acc_stderr": 0.024127463462650153, + "brier_score": 0.8660242939475324, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_professional_psychology": { - "acc": 0.26143790849673204, - "acc_stderr": 0.017776947157528037, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.272875816993464, + "acc_stderr": 0.01802047414839358, + "brier_score": 0.785573099759002, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_human_sexuality": { - "acc": 0.20610687022900764, - "acc_stderr": 0.03547771004159463, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.1984732824427481, + "acc_stderr": 0.03498149385462471, + "brier_score": 0.8018306923112876, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_econometrics": { - "acc": 0.2543859649122807, - "acc_stderr": 0.04096985139843671, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.19298245614035087, + "acc_stderr": 0.037124548537213684, + "brier_score": 0.80948299531357, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_miscellaneous": { - "acc": 0.27330779054916987, - "acc_stderr": 0.015936681062628556, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.29118773946360155, + "acc_stderr": 0.016246087069701404, + "brier_score": 0.7720002992526842, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_marketing": { - "acc": 0.31196581196581197, - "acc_stderr": 0.030351527323344944, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.2777777777777778, + "acc_stderr": 0.029343114798094462, + "brier_score": 0.7700657137319908, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_management": { - "acc": 0.1262135922330097, - "acc_stderr": 0.03288180278808628, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.24271844660194175, + "acc_stderr": 0.04245022486384495, + "brier_score": 0.8236806321437848, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_nutrition": { - "acc": 0.238562091503268, - "acc_stderr": 0.02440439492808787, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.27450980392156865, + "acc_stderr": 0.025553169991826528, + "brier_score": 0.819694635782886, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_medical_genetics": { - "acc": 0.23, - "acc_stderr": 0.04229525846816505, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.26, + "acc_stderr": 0.04408440022768076, + "brier_score": 0.7874899117454538, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_human_aging": { - "acc": 0.32286995515695066, - "acc_stderr": 0.03138147637575498, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.29596412556053814, + "acc_stderr": 0.0306365913486998, + "brier_score": 0.7483159620278041, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_professional_medicine": { - "acc": 0.35661764705882354, - "acc_stderr": 0.029097209568411955, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.20220588235294118, + "acc_stderr": 0.024398192986654924, + "brier_score": 0.8288349039405379, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_college_medicine": { "acc": 0.23699421965317918, "acc_stderr": 0.03242414757483098, - "timestamp": "2024-11-11T08-57-26.989242" + "brier_score": 0.8334233429462069, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_business_ethics": { - "acc": 0.21, - "acc_stderr": 0.040936018074033256, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.29, + "acc_stderr": 0.045604802157206845, + "brier_score": 0.7715835120318814, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_clinical_knowledge": { - "acc": 0.25660377358490566, - "acc_stderr": 0.026880647889051982, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.2528301886792453, + "acc_stderr": 0.02674989977124124, + "brier_score": 0.8192907779010937, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_global_facts": { - "acc": 0.32, - "acc_stderr": 0.04688261722621505, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 0.7677745198062343, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_virology": { - "acc": 0.29518072289156627, - "acc_stderr": 0.035509201856896294, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.23493975903614459, + "acc_stderr": 0.03300533186128922, + "brier_score": 0.7896623984697541, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_professional_accounting": { - "acc": 0.2765957446808511, - "acc_stderr": 0.02668456434046098, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.23404255319148937, + "acc_stderr": 0.025257861359432403, + "brier_score": 0.8035852632894487, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_college_physics": { - "acc": 0.21568627450980393, - "acc_stderr": 0.040925639582376556, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.11764705882352941, + "acc_stderr": 0.03205907733144527, + "brier_score": 0.7963115355798652, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_high_school_physics": { "acc": 0.26490066225165565, "acc_stderr": 0.036030385453603826, - "timestamp": "2024-11-11T08-57-26.989242" + "brier_score": 0.7843686953032997, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_high_school_biology": { - "acc": 0.3258064516129032, - "acc_stderr": 0.026662010578567104, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.2645161290322581, + "acc_stderr": 0.025091892378859275, + "brier_score": 0.7824577489394099, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_college_biology": { - "acc": 0.2638888888888889, - "acc_stderr": 0.03685651095897532, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.2361111111111111, + "acc_stderr": 0.03551446610810826, + "brier_score": 0.7743808593997604, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_anatomy": { - "acc": 0.3037037037037037, - "acc_stderr": 0.039725528847851375, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.2962962962962963, + "acc_stderr": 0.03944624162501116, + "brier_score": 0.7409161817727408, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_college_chemistry": { - "acc": 0.13, - "acc_stderr": 0.03379976689896309, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.18, + "acc_stderr": 0.038612291966536955, + "brier_score": 0.8176881871196044, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_computer_security": { - "acc": 0.23, - "acc_stderr": 0.04229525846816507, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.29, + "acc_stderr": 0.045604802157206845, + "brier_score": 0.7562892819901279, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_college_computer_science": { - "acc": 0.37, - "acc_stderr": 0.048523658709391, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.29, + "acc_stderr": 0.045604802157206845, + "brier_score": 0.7794711889789567, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_astronomy": { - "acc": 0.19078947368421054, - "acc_stderr": 0.031975658210325, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.23684210526315788, + "acc_stderr": 0.03459777606810537, + "brier_score": 0.7763864267043885, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_college_mathematics": { - "acc": 0.23, - "acc_stderr": 0.04229525846816506, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.29, + "acc_stderr": 0.04560480215720684, + "brier_score": 0.7686642085054471, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_conceptual_physics": { - "acc": 0.2553191489361702, - "acc_stderr": 0.028504856470514203, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.23404255319148937, + "acc_stderr": 0.0276784525782124, + "brier_score": 0.7660169929396504, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_abstract_algebra": { - "acc": 0.21, - "acc_stderr": 0.040936018074033256, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.26, + "acc_stderr": 0.0440844002276808, + "brier_score": 0.7865734465162898, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_high_school_computer_science": { - "acc": 0.33, - "acc_stderr": 0.04725815626252605, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.38, + "acc_stderr": 0.048783173121456316, + "brier_score": 0.7439680983789182, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_machine_learning": { - "acc": 0.22321428571428573, - "acc_stderr": 0.039523019677025116, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.3392857142857143, + "acc_stderr": 0.04493949068613539, + "brier_score": 0.7405766796738416, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_high_school_chemistry": { - "acc": 0.22167487684729065, - "acc_stderr": 0.029225575892489614, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.2019704433497537, + "acc_stderr": 0.02824735012218027, + "brier_score": 0.7965945037147942, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_high_school_statistics": { - "acc": 0.44907407407407407, - "acc_stderr": 0.03392238405321616, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.1712962962962963, + "acc_stderr": 0.025695341643824688, + "brier_score": 0.8460945729720145, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_elementary_mathematics": { - "acc": 0.2619047619047619, - "acc_stderr": 0.022644212615525218, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.2671957671957672, + "acc_stderr": 0.02278967314577656, + "brier_score": 0.8122475508499548, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_electrical_engineering": { - "acc": 0.2827586206896552, - "acc_stderr": 0.03752833958003336, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.3103448275862069, + "acc_stderr": 0.03855289616378949, + "brier_score": 0.7545684510084733, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "mmlu_high_school_mathematics": { - "acc": 0.27037037037037037, - "acc_stderr": 0.027080372815145665, - "timestamp": "2024-11-11T08-57-26.989242" + "acc": 0.25555555555555554, + "acc_stderr": 0.02659393910184408, + "brier_score": 0.8056289075921087, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-11-53.465958" }, "arc_challenge": { "acc": 0.22184300341296928, diff --git a/metadata/model_scores/results_Salesforce__codegen-350M-mono.json b/metadata/model_scores/results_Salesforce__codegen-350M-mono.json index 9953ccf..bad67d5 100644 --- a/metadata/model_scores/results_Salesforce__codegen-350M-mono.json +++ b/metadata/model_scores/results_Salesforce__codegen-350M-mono.json @@ -1,6 +1,6 @@ { "model_name": "Salesforce/codegen-350M-mono", - "last_updated": "2024-12-04 11:24:41.286914", + "last_updated": "2024-12-19 13:40:35.085267", "results": { "harness": { "minerva_math_precalc": { @@ -230,287 +230,401 @@ "mmlu_world_religions": { "acc": 0.29239766081871343, "acc_stderr": 0.03488647713457922, - "timestamp": "2024-11-12T18-20-13.928795" + "brier_score": 0.8396180714465893, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_formal_logic": { - "acc": 0.21428571428571427, - "acc_stderr": 0.036700664510471825, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.2777777777777778, + "acc_stderr": 0.04006168083848877, + "brier_score": 0.8348683170976435, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_prehistory": { - "acc": 0.20679012345679013, - "acc_stderr": 0.022535006705942818, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.2191358024691358, + "acc_stderr": 0.023016705640262213, + "brier_score": 0.898568305536742, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_moral_scenarios": { - "acc": 0.23798882681564246, - "acc_stderr": 0.014242630070574885, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.25251396648044694, + "acc_stderr": 0.01453033020146864, + "brier_score": 0.797675676064862, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_high_school_world_history": { - "acc": 0.2320675105485232, - "acc_stderr": 0.027479744550808514, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.26582278481012656, + "acc_stderr": 0.028756799629658335, + "brier_score": 0.8232921151238497, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_moral_disputes": { - "acc": 0.18497109826589594, - "acc_stderr": 0.020903975842083033, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.2630057803468208, + "acc_stderr": 0.023703099525258165, + "brier_score": 0.8652584497055802, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_professional_law": { - "acc": 0.24902216427640156, - "acc_stderr": 0.01104489226404077, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.2457627118644068, + "acc_stderr": 0.01099615663514269, + "brier_score": 0.8576270672615085, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_logical_fallacies": { - "acc": 0.25766871165644173, - "acc_stderr": 0.03436150827846917, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.22699386503067484, + "acc_stderr": 0.032910995786157686, + "brier_score": 0.9149466079655856, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_high_school_us_history": { - "acc": 0.25980392156862747, - "acc_stderr": 0.030778554678693264, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.24019607843137256, + "acc_stderr": 0.02998373305591361, + "brier_score": 0.8328847382132503, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_philosophy": { - "acc": 0.24115755627009647, - "acc_stderr": 0.02429659403476343, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.19292604501607716, + "acc_stderr": 0.022411516780911363, + "brier_score": 1.0091701908373931, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_jurisprudence": { - "acc": 0.24074074074074073, - "acc_stderr": 0.04133119440243838, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.2962962962962963, + "acc_stderr": 0.04414343666854933, + "brier_score": 0.8145598918889535, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_international_law": { - "acc": 0.24793388429752067, - "acc_stderr": 0.03941897526516303, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.23140495867768596, + "acc_stderr": 0.03849856098794088, + "brier_score": 0.9721588320326391, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_high_school_european_history": { - "acc": 0.24242424242424243, - "acc_stderr": 0.03346409881055953, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.23636363636363636, + "acc_stderr": 0.03317505930009181, + "brier_score": 0.8278142947763675, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_high_school_government_and_politics": { - "acc": 0.3626943005181347, - "acc_stderr": 0.03469713791704372, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.20207253886010362, + "acc_stderr": 0.02897908979429673, + "brier_score": 0.959848409778557, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_high_school_microeconomics": { - "acc": 0.2647058823529412, - "acc_stderr": 0.028657491285071973, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.21008403361344538, + "acc_stderr": 0.026461398717471874, + "brier_score": 0.928203227541235, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_high_school_geography": { - "acc": 0.35353535353535354, - "acc_stderr": 0.03406086723547153, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.18181818181818182, + "acc_stderr": 0.027479603010538804, + "brier_score": 0.9445360520364772, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_high_school_psychology": { - "acc": 0.30091743119266057, - "acc_stderr": 0.019664751366802114, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.2, + "acc_stderr": 0.01714985851425095, + "brier_score": 0.9262547542438837, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_public_relations": { "acc": 0.20909090909090908, - "acc_stderr": 0.03895091015724138, - "timestamp": "2024-11-12T18-20-13.928795" + "acc_stderr": 0.038950910157241364, + "brier_score": 0.872707172893249, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_us_foreign_policy": { - "acc": 0.27, - "acc_stderr": 0.0446196043338474, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.29, + "acc_stderr": 0.045604802157206845, + "brier_score": 0.8664993053477293, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_sociology": { - "acc": 0.23880597014925373, - "acc_stderr": 0.030147775935409224, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.24378109452736318, + "acc_stderr": 0.030360490154014652, + "brier_score": 1.0189674683196024, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_high_school_macroeconomics": { - "acc": 0.33589743589743587, - "acc_stderr": 0.023946724741563976, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.2230769230769231, + "acc_stderr": 0.021107730127243995, + "brier_score": 0.9392181616810883, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_security_studies": { - "acc": 0.27346938775510204, - "acc_stderr": 0.02853556033712845, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.17551020408163265, + "acc_stderr": 0.024352800722970015, + "brier_score": 0.9534796271553114, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_professional_psychology": { - "acc": 0.2434640522875817, - "acc_stderr": 0.017362473762146644, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.272875816993464, + "acc_stderr": 0.018020474148393577, + "brier_score": 0.9046486802299831, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_human_sexuality": { - "acc": 0.2366412213740458, - "acc_stderr": 0.03727673575596918, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.22900763358778625, + "acc_stderr": 0.036853466317118506, + "brier_score": 0.9188173589415536, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_econometrics": { - "acc": 0.21052631578947367, - "acc_stderr": 0.0383515395439942, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.23684210526315788, + "acc_stderr": 0.039994238792813365, + "brier_score": 0.8627957595734717, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_miscellaneous": { - "acc": 0.25287356321839083, - "acc_stderr": 0.01554337731371968, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.2515964240102171, + "acc_stderr": 0.015517322365529631, + "brier_score": 0.8501331895844128, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_marketing": { - "acc": 0.23931623931623933, - "acc_stderr": 0.027951826808924333, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.28205128205128205, + "acc_stderr": 0.02948036054954119, + "brier_score": 0.8850419108960376, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_management": { - "acc": 0.20388349514563106, - "acc_stderr": 0.03989139859531772, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.17475728155339806, + "acc_stderr": 0.03760178006026621, + "brier_score": 0.9488170330428574, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_nutrition": { - "acc": 0.2549019607843137, - "acc_stderr": 0.024954184324879912, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.26143790849673204, + "acc_stderr": 0.025160998214292456, + "brier_score": 0.8710136973501563, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_medical_genetics": { - "acc": 0.3, - "acc_stderr": 0.046056618647183814, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.29, + "acc_stderr": 0.04560480215720684, + "brier_score": 0.8155383796623732, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_human_aging": { - "acc": 0.21973094170403587, - "acc_stderr": 0.027790177064383602, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.3273542600896861, + "acc_stderr": 0.03149384670994131, + "brier_score": 0.8526030027294819, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_professional_medicine": { - "acc": 0.3602941176470588, - "acc_stderr": 0.029163128570670733, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.20220588235294118, + "acc_stderr": 0.024398192986654924, + "brier_score": 0.8277778329217774, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_college_medicine": { - "acc": 0.2543352601156069, - "acc_stderr": 0.0332055644308557, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.20809248554913296, + "acc_stderr": 0.030952890217749895, + "brier_score": 0.9308911708837603, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_business_ethics": { - "acc": 0.33, - "acc_stderr": 0.04725815626252604, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "brier_score": 0.8362462742598932, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_clinical_knowledge": { - "acc": 0.2792452830188679, - "acc_stderr": 0.027611163402399715, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.19622641509433963, + "acc_stderr": 0.02444238813110082, + "brier_score": 0.9179332108618161, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_global_facts": { - "acc": 0.18, - "acc_stderr": 0.038612291966536955, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.23, + "acc_stderr": 0.042295258468165065, + "brier_score": 0.8688533991922952, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_virology": { - "acc": 0.1927710843373494, - "acc_stderr": 0.03070982405056527, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.28313253012048195, + "acc_stderr": 0.03507295431370518, + "brier_score": 0.8792122675212317, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_professional_accounting": { "acc": 0.2765957446808511, "acc_stderr": 0.026684564340460997, - "timestamp": "2024-11-12T18-20-13.928795" + "brier_score": 0.8605899062455952, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_college_physics": { - "acc": 0.20588235294117646, - "acc_stderr": 0.040233822736177476, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.2549019607843137, + "acc_stderr": 0.0433643270799318, + "brier_score": 0.8501212667575395, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_high_school_physics": { - "acc": 0.2781456953642384, - "acc_stderr": 0.03658603262763743, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.26490066225165565, + "acc_stderr": 0.036030385453603826, + "brier_score": 0.8608175490841444, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_high_school_biology": { - "acc": 0.29354838709677417, - "acc_stderr": 0.02590608702131929, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.20967741935483872, + "acc_stderr": 0.023157879349083532, + "brier_score": 0.8865733013303182, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_college_biology": { - "acc": 0.25, - "acc_stderr": 0.03621034121889507, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.2638888888888889, + "acc_stderr": 0.03685651095897532, + "brier_score": 0.8503616280461068, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_anatomy": { - "acc": 0.24444444444444444, - "acc_stderr": 0.037125378336148665, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.2, + "acc_stderr": 0.03455473702325437, + "brier_score": 0.8940578241939349, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_college_chemistry": { - "acc": 0.24, - "acc_stderr": 0.042923469599092816, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.18, + "acc_stderr": 0.03861229196653695, + "brier_score": 0.8989931116226676, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_computer_security": { - "acc": 0.21, - "acc_stderr": 0.040936018074033256, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.29, + "acc_stderr": 0.045604802157206845, + "brier_score": 0.8061276351309151, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_college_computer_science": { - "acc": 0.38, - "acc_stderr": 0.048783173121456316, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.23, + "acc_stderr": 0.04229525846816505, + "brier_score": 0.8625106698190292, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_astronomy": { - "acc": 0.18421052631578946, - "acc_stderr": 0.0315469804508223, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.19078947368421054, + "acc_stderr": 0.031975658210325, + "brier_score": 0.8972433159767368, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_college_mathematics": { - "acc": 0.26, - "acc_stderr": 0.0440844002276808, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.24, + "acc_stderr": 0.04292346959909282, + "brier_score": 0.8340920973781174, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_conceptual_physics": { - "acc": 0.33617021276595743, - "acc_stderr": 0.030881618520676942, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.24680851063829787, + "acc_stderr": 0.028185441301234116, + "brier_score": 0.8605806596292578, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_abstract_algebra": { - "acc": 0.21, - "acc_stderr": 0.040936018074033256, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.22, + "acc_stderr": 0.04163331998932269, + "brier_score": 0.8899781796093954, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_high_school_computer_science": { - "acc": 0.28, - "acc_stderr": 0.04512608598542127, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.29, + "acc_stderr": 0.045604802157206845, + "brier_score": 0.837163754480655, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_machine_learning": { - "acc": 0.25892857142857145, - "acc_stderr": 0.04157751539865629, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.26785714285714285, + "acc_stderr": 0.04203277291467762, + "brier_score": 0.8136091820043374, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_high_school_chemistry": { - "acc": 0.28078817733990147, - "acc_stderr": 0.03161856335358609, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.20689655172413793, + "acc_stderr": 0.028501378167893946, + "brier_score": 0.9127406347974003, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_high_school_statistics": { - "acc": 0.4444444444444444, - "acc_stderr": 0.03388857118502326, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.19907407407407407, + "acc_stderr": 0.027232298462690232, + "brier_score": 0.9278457068781704, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_elementary_mathematics": { "acc": 0.25132275132275134, "acc_stderr": 0.022340482339643895, - "timestamp": "2024-11-12T18-20-13.928795" + "brier_score": 0.8694753783996194, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_electrical_engineering": { "acc": 0.2482758620689655, "acc_stderr": 0.03600105692727771, - "timestamp": "2024-11-12T18-20-13.928795" + "brier_score": 0.8792815661151909, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "mmlu_high_school_mathematics": { - "acc": 0.2518518518518518, - "acc_stderr": 0.026466117538959905, - "timestamp": "2024-11-12T18-20-13.928795" + "acc": 0.24444444444444444, + "acc_stderr": 0.02620276653465215, + "brier_score": 0.8594830956920774, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-11-02.066995" }, "arc_challenge": { "acc": 0.17491467576791808, diff --git a/metadata/model_scores/results_Salesforce__codegen-350M-multi.json b/metadata/model_scores/results_Salesforce__codegen-350M-multi.json index b55c9d5..e41168d 100644 --- a/metadata/model_scores/results_Salesforce__codegen-350M-multi.json +++ b/metadata/model_scores/results_Salesforce__codegen-350M-multi.json @@ -1,6 +1,6 @@ { "model_name": "Salesforce/codegen-350M-multi", - "last_updated": "2024-12-04 11:23:50.264229", + "last_updated": "2024-12-19 13:39:13.209588", "results": { "harness": { "minerva_math_precalc": { @@ -228,289 +228,403 @@ "timestamp": "2024-06-13T18-22-36.360059" }, "mmlu_world_religions": { - "acc": 0.25146198830409355, - "acc_stderr": 0.033275044238468436, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.3157894736842105, + "acc_stderr": 0.03565079670708313, + "brier_score": 0.9076127637219964, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_formal_logic": { - "acc": 0.18253968253968253, - "acc_stderr": 0.03455071019102147, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.2619047619047619, + "acc_stderr": 0.03932537680392872, + "brier_score": 0.8460958087423824, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_prehistory": { - "acc": 0.22839506172839505, - "acc_stderr": 0.023358211840626267, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.2222222222222222, + "acc_stderr": 0.023132376234543325, + "brier_score": 0.987859802666584, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_moral_scenarios": { - "acc": 0.2424581005586592, - "acc_stderr": 0.014333522059217889, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.2446927374301676, + "acc_stderr": 0.01437816988409841, + "brier_score": 0.7630963566676106, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_high_school_world_history": { - "acc": 0.2911392405063291, - "acc_stderr": 0.029571601065753374, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.26582278481012656, + "acc_stderr": 0.028756799629658335, + "brier_score": 0.9634761184938231, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_moral_disputes": { - "acc": 0.23410404624277456, - "acc_stderr": 0.022797110278071134, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.24566473988439305, + "acc_stderr": 0.02317629820399201, + "brier_score": 0.9605339536210215, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_professional_law": { - "acc": 0.2470664928292047, - "acc_stderr": 0.011015752255279329, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.2438070404172099, + "acc_stderr": 0.010966507972178475, + "brier_score": 0.8748290864284843, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_logical_fallacies": { - "acc": 0.2883435582822086, - "acc_stderr": 0.035590395316173425, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.22085889570552147, + "acc_stderr": 0.032591773927421776, + "brier_score": 1.044997786061394, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_high_school_us_history": { - "acc": 0.27450980392156865, - "acc_stderr": 0.03132179803083289, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.2549019607843137, + "acc_stderr": 0.030587591351604246, + "brier_score": 0.9482436683910561, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_philosophy": { - "acc": 0.2572347266881029, - "acc_stderr": 0.024826171289250888, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.1864951768488746, + "acc_stderr": 0.02212243977248077, + "brier_score": 1.0453732068606783, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_jurisprudence": { - "acc": 0.23148148148148148, - "acc_stderr": 0.04077494709252626, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.26851851851851855, + "acc_stderr": 0.04284467968052192, + "brier_score": 0.9867169959007726, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_international_law": { - "acc": 0.2809917355371901, - "acc_stderr": 0.04103203830514512, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.2396694214876033, + "acc_stderr": 0.03896878985070417, + "brier_score": 1.0114064863859569, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_high_school_european_history": { - "acc": 0.2909090909090909, - "acc_stderr": 0.03546563019624336, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.20606060606060606, + "acc_stderr": 0.0315841532404771, + "brier_score": 0.9819375183718658, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_high_school_government_and_politics": { - "acc": 0.36787564766839376, - "acc_stderr": 0.034801756684660366, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.19689119170984457, + "acc_stderr": 0.02869787397186069, + "brier_score": 1.016578117624572, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_high_school_microeconomics": { - "acc": 0.21428571428571427, - "acc_stderr": 0.026653531596715473, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.20168067226890757, + "acc_stderr": 0.026064313406304527, + "brier_score": 1.0361993540193473, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_high_school_geography": { - "acc": 0.3484848484848485, - "acc_stderr": 0.033948539651564025, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.1717171717171717, + "acc_stderr": 0.026869716187429917, + "brier_score": 1.0463816816619436, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_high_school_psychology": { - "acc": 0.30642201834862387, - "acc_stderr": 0.019765517220458523, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.1908256880733945, + "acc_stderr": 0.01684767640009109, + "brier_score": 1.108188168555968, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_public_relations": { - "acc": 0.14545454545454545, - "acc_stderr": 0.0337689831983308, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.21818181818181817, + "acc_stderr": 0.03955932861795833, + "brier_score": 1.07147508063318, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_us_foreign_policy": { - "acc": 0.26, - "acc_stderr": 0.0440844002276808, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.28, + "acc_stderr": 0.045126085985421276, + "brier_score": 0.8866703263248481, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_sociology": { - "acc": 0.23383084577114427, - "acc_stderr": 0.029929415408348384, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.24378109452736318, + "acc_stderr": 0.030360490154014652, + "brier_score": 1.0909873671496528, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_high_school_macroeconomics": { - "acc": 0.32564102564102565, - "acc_stderr": 0.02375966576741229, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.2076923076923077, + "acc_stderr": 0.02056753956724679, + "brier_score": 0.9714959985838358, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_security_studies": { - "acc": 0.30612244897959184, - "acc_stderr": 0.02950489645459596, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.19591836734693877, + "acc_stderr": 0.025409301953225678, + "brier_score": 1.035349559854223, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_professional_psychology": { - "acc": 0.2549019607843137, - "acc_stderr": 0.017630827375148383, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.24836601307189543, + "acc_stderr": 0.017479487001364764, + "brier_score": 1.0612778969181413, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_human_sexuality": { - "acc": 0.24427480916030533, - "acc_stderr": 0.03768335959728743, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.25190839694656486, + "acc_stderr": 0.038073871163060866, + "brier_score": 1.0156352887399498, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_econometrics": { "acc": 0.23684210526315788, - "acc_stderr": 0.039994238792813344, - "timestamp": "2024-11-21T23-05-09.706261" + "acc_stderr": 0.03999423879281338, + "brier_score": 0.9444932309114732, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_miscellaneous": { - "acc": 0.24521072796934865, - "acc_stderr": 0.01538435228454394, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.23627075351213284, + "acc_stderr": 0.01519047371703751, + "brier_score": 0.9882763883266693, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_marketing": { - "acc": 0.19658119658119658, - "acc_stderr": 0.02603538609895129, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.2905982905982906, + "acc_stderr": 0.029745048572674057, + "brier_score": 1.0379668720217567, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_management": { - "acc": 0.2621359223300971, - "acc_stderr": 0.04354631077260595, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.17475728155339806, + "acc_stderr": 0.03760178006026621, + "brier_score": 1.14546352508103, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_nutrition": { - "acc": 0.26143790849673204, - "acc_stderr": 0.025160998214292456, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.21895424836601307, + "acc_stderr": 0.02367908986180772, + "brier_score": 1.0325525503748743, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_medical_genetics": { - "acc": 0.3, - "acc_stderr": 0.046056618647183814, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "brier_score": 0.9153923451609859, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_human_aging": { - "acc": 0.2825112107623318, - "acc_stderr": 0.030216831011508755, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.31390134529147984, + "acc_stderr": 0.03114679648297246, + "brier_score": 0.9091966675496221, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_professional_medicine": { - "acc": 0.4522058823529412, - "acc_stderr": 0.030233758551596445, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.18382352941176472, + "acc_stderr": 0.02352924218519311, + "brier_score": 1.041393277404226, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_college_medicine": { - "acc": 0.27167630057803466, - "acc_stderr": 0.03391750322321659, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.20809248554913296, + "acc_stderr": 0.030952890217749884, + "brier_score": 1.035016973705624, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_business_ethics": { - "acc": 0.31, - "acc_stderr": 0.04648231987117316, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "brier_score": 0.949501669515644, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_clinical_knowledge": { - "acc": 0.2830188679245283, - "acc_stderr": 0.027724236492700907, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.22264150943396227, + "acc_stderr": 0.025604233470899088, + "brier_score": 1.0155779930799764, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_global_facts": { - "acc": 0.16, - "acc_stderr": 0.036845294917747094, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.18, + "acc_stderr": 0.03861229196653697, + "brier_score": 0.9658862144878719, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_virology": { - "acc": 0.1686746987951807, - "acc_stderr": 0.029152009627856544, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.28313253012048195, + "acc_stderr": 0.03507295431370518, + "brier_score": 0.9147287745722621, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_professional_accounting": { - "acc": 0.22695035460992907, - "acc_stderr": 0.024987106365642976, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.23049645390070922, + "acc_stderr": 0.025123739226872405, + "brier_score": 0.972198891754218, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_college_physics": { - "acc": 0.22549019607843138, - "acc_stderr": 0.04158307533083286, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.21568627450980393, + "acc_stderr": 0.040925639582376556, + "brier_score": 0.958858133650472, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_high_school_physics": { - "acc": 0.2781456953642384, - "acc_stderr": 0.03658603262763743, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.2119205298013245, + "acc_stderr": 0.03336767086567978, + "brier_score": 0.9895118534031518, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_high_school_biology": { - "acc": 0.27419354838709675, - "acc_stderr": 0.0253781399708852, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.18064516129032257, + "acc_stderr": 0.021886178567172558, + "brier_score": 1.0620215312594377, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_college_biology": { - "acc": 0.20833333333333334, - "acc_stderr": 0.033961162058453336, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.2569444444444444, + "acc_stderr": 0.03653946969442099, + "brier_score": 0.9572646773476323, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_anatomy": { - "acc": 0.2518518518518518, - "acc_stderr": 0.03749850709174023, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.1925925925925926, + "acc_stderr": 0.03406542058502653, + "brier_score": 1.0196718677574592, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_college_chemistry": { - "acc": 0.29, - "acc_stderr": 0.045604802157206845, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.19, + "acc_stderr": 0.03942772444036623, + "brier_score": 0.9834123630567483, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_computer_security": { - "acc": 0.23, - "acc_stderr": 0.04229525846816506, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.28, + "acc_stderr": 0.045126085985421276, + "brier_score": 0.9511809132462804, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_college_computer_science": { - "acc": 0.29, - "acc_stderr": 0.045604802157206845, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 0.971996928215002, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_astronomy": { - "acc": 0.17763157894736842, - "acc_stderr": 0.031103182383123394, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.17105263157894737, + "acc_stderr": 0.03064360707167709, + "brier_score": 1.0437098022366331, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_college_mathematics": { - "acc": 0.25, - "acc_stderr": 0.04351941398892446, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "brier_score": 0.966038248892535, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_conceptual_physics": { - "acc": 0.2425531914893617, - "acc_stderr": 0.028020226271200217, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.26382978723404255, + "acc_stderr": 0.02880998985410298, + "brier_score": 0.9290328394695287, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_abstract_algebra": { - "acc": 0.23, - "acc_stderr": 0.042295258468165065, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "brier_score": 1.038614939773198, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_high_school_computer_science": { - "acc": 0.19, - "acc_stderr": 0.03942772444036623, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.23, + "acc_stderr": 0.04229525846816506, + "brier_score": 0.9759436161616507, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_machine_learning": { "acc": 0.33035714285714285, "acc_stderr": 0.04464285714285712, - "timestamp": "2024-11-21T23-05-09.706261" + "brier_score": 0.8620202101436615, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_high_school_chemistry": { - "acc": 0.21674876847290642, - "acc_stderr": 0.028990331252516235, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.1724137931034483, + "acc_stderr": 0.02657767218303657, + "brier_score": 1.0588217991363558, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_high_school_statistics": { - "acc": 0.46296296296296297, - "acc_stderr": 0.034006036255382704, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.1388888888888889, + "acc_stderr": 0.02358544736890012, + "brier_score": 1.0655089115094414, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_elementary_mathematics": { - "acc": 0.2566137566137566, - "acc_stderr": 0.022494510767503154, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.2222222222222222, + "acc_stderr": 0.021411684393694175, + "brier_score": 0.9576128764995636, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_electrical_engineering": { - "acc": 0.27586206896551724, - "acc_stderr": 0.037245636197746304, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.25517241379310346, + "acc_stderr": 0.03632984052707842, + "brier_score": 0.9581270488425577, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "mmlu_high_school_mathematics": { - "acc": 0.2814814814814815, - "acc_stderr": 0.02742001935094527, - "timestamp": "2024-11-21T23-05-09.706261" + "acc": 0.21481481481481482, + "acc_stderr": 0.025040443877000683, + "brier_score": 1.0486529075001332, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-58-20.110786" }, "arc_challenge": { "acc": 0.17320819112627986, diff --git a/metadata/model_scores/results_Salesforce__codegen-6B-mono.json b/metadata/model_scores/results_Salesforce__codegen-6B-mono.json index 25c25b9..39612b9 100644 --- a/metadata/model_scores/results_Salesforce__codegen-6B-mono.json +++ b/metadata/model_scores/results_Salesforce__codegen-6B-mono.json @@ -1,6 +1,6 @@ { "model_name": "Salesforce/codegen-6B-mono", - "last_updated": "2024-12-04 11:24:32.569700", + "last_updated": "2024-12-19 13:40:26.028990", "results": { "harness": { "minerva_math_precalc": { @@ -228,289 +228,403 @@ "timestamp": "2024-06-14T01-05-27.879801" }, "mmlu_world_religions": { - "acc": 0.29239766081871343, - "acc_stderr": 0.034886477134579215, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.3157894736842105, + "acc_stderr": 0.03565079670708312, + "brier_score": 0.7780453162294215, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_formal_logic": { - "acc": 0.29365079365079366, - "acc_stderr": 0.04073524322147127, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.3412698412698413, + "acc_stderr": 0.04240799327574924, + "brier_score": 0.7584902825096467, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_prehistory": { - "acc": 0.2808641975308642, - "acc_stderr": 0.025006469755799208, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.21604938271604937, + "acc_stderr": 0.022899162918445806, + "brier_score": 0.8522473735930954, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_moral_scenarios": { - "acc": 0.24692737430167597, - "acc_stderr": 0.014422292204808836, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.23798882681564246, + "acc_stderr": 0.014242630070574885, + "brier_score": 0.8018951873309403, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_high_school_world_history": { - "acc": 0.25738396624472576, - "acc_stderr": 0.028458820991460285, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.270042194092827, + "acc_stderr": 0.028900721906293426, + "brier_score": 0.8322130716486043, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_moral_disputes": { - "acc": 0.25722543352601157, - "acc_stderr": 0.023532925431044283, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.2514450867052023, + "acc_stderr": 0.023357365785874037, + "brier_score": 0.8062744853068, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_professional_law": { - "acc": 0.24185136897001303, - "acc_stderr": 0.010936550813827054, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.24511082138200782, + "acc_stderr": 0.010986307870045509, + "brier_score": 0.8031523145939087, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_logical_fallacies": { - "acc": 0.27607361963190186, - "acc_stderr": 0.0351238528370505, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.2147239263803681, + "acc_stderr": 0.03226219377286774, + "brier_score": 0.8443732327706741, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_high_school_us_history": { - "acc": 0.24019607843137256, - "acc_stderr": 0.02998373305591361, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.25, + "acc_stderr": 0.03039153369274154, + "brier_score": 0.8459099121028086, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_philosophy": { - "acc": 0.2282958199356913, - "acc_stderr": 0.02383930331139822, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.19292604501607716, + "acc_stderr": 0.022411516780911363, + "brier_score": 0.8363464662368304, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_jurisprudence": { - "acc": 0.28703703703703703, - "acc_stderr": 0.043733130409147614, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.26851851851851855, + "acc_stderr": 0.04284467968052192, + "brier_score": 0.7999006000971114, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_international_law": { - "acc": 0.256198347107438, - "acc_stderr": 0.03984979653302872, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.24793388429752067, + "acc_stderr": 0.03941897526516303, + "brier_score": 0.8350664431932452, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_high_school_european_history": { - "acc": 0.23636363636363636, - "acc_stderr": 0.033175059300091805, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.20606060606060606, + "acc_stderr": 0.0315841532404771, + "brier_score": 0.8866890339705624, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_high_school_government_and_politics": { - "acc": 0.23834196891191708, - "acc_stderr": 0.03074890536390989, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.19689119170984457, + "acc_stderr": 0.02869787397186069, + "brier_score": 0.8399415509436173, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_high_school_microeconomics": { - "acc": 0.3277310924369748, - "acc_stderr": 0.03048991141767323, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.20168067226890757, + "acc_stderr": 0.026064313406304527, + "brier_score": 0.8356456450973151, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_high_school_geography": { - "acc": 0.26262626262626265, - "acc_stderr": 0.031353050095330834, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.17676767676767677, + "acc_stderr": 0.027178752639044915, + "brier_score": 0.8601059320404222, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_high_school_psychology": { - "acc": 0.23302752293577983, - "acc_stderr": 0.018125669180861496, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.1944954128440367, + "acc_stderr": 0.01697028909045803, + "brier_score": 0.8793552688694627, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_public_relations": { - "acc": 0.19090909090909092, - "acc_stderr": 0.03764425585984926, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.21818181818181817, + "acc_stderr": 0.03955932861795833, + "brier_score": 0.8392366871864418, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_us_foreign_policy": { - "acc": 0.33, - "acc_stderr": 0.04725815626252604, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.26, + "acc_stderr": 0.044084400227680794, + "brier_score": 0.8079258933629248, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_sociology": { - "acc": 0.263681592039801, - "acc_stderr": 0.03115715086935557, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.24875621890547264, + "acc_stderr": 0.030567675938916718, + "brier_score": 0.8280539163246622, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_high_school_macroeconomics": { - "acc": 0.28717948717948716, - "acc_stderr": 0.022939925418530616, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.20512820512820512, + "acc_stderr": 0.020473233173551965, + "brier_score": 0.8476793974498708, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_security_studies": { - "acc": 0.2857142857142857, - "acc_stderr": 0.028920583220675578, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.19183673469387755, + "acc_stderr": 0.025206963154225395, + "brier_score": 0.8042266307100163, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_professional_psychology": { - "acc": 0.2679738562091503, - "acc_stderr": 0.017917974069594726, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.24836601307189543, + "acc_stderr": 0.017479487001364764, + "brier_score": 0.8331333456314579, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_human_sexuality": { - "acc": 0.25190839694656486, - "acc_stderr": 0.03807387116306087, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.2595419847328244, + "acc_stderr": 0.03844876139785271, + "brier_score": 0.8131038913189726, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_econometrics": { - "acc": 0.24561403508771928, - "acc_stderr": 0.04049339297748142, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.23684210526315788, + "acc_stderr": 0.03999423879281338, + "brier_score": 0.81680580181963, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_miscellaneous": { - "acc": 0.2681992337164751, - "acc_stderr": 0.015842430835269428, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.23754789272030652, + "acc_stderr": 0.015218733046150193, + "brier_score": 0.8326763568305254, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_marketing": { - "acc": 0.25213675213675213, - "acc_stderr": 0.02844796547623101, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.2863247863247863, + "acc_stderr": 0.02961432369045666, + "brier_score": 0.7995292177441591, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_management": { - "acc": 0.1553398058252427, - "acc_stderr": 0.035865947385739734, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.17475728155339806, + "acc_stderr": 0.03760178006026621, + "brier_score": 0.8966846263949297, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_nutrition": { - "acc": 0.2549019607843137, - "acc_stderr": 0.02495418432487991, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.22875816993464052, + "acc_stderr": 0.024051029739912255, + "brier_score": 0.8368563375087384, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_medical_genetics": { - "acc": 0.25, - "acc_stderr": 0.04351941398892446, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "brier_score": 0.7830689974921099, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_human_aging": { - "acc": 0.25112107623318386, - "acc_stderr": 0.02910522083322461, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.3094170403587444, + "acc_stderr": 0.031024411740572213, + "brier_score": 0.7777714690289778, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_professional_medicine": { - "acc": 0.3786764705882353, - "acc_stderr": 0.029465133639776125, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.1875, + "acc_stderr": 0.023709788253811766, + "brier_score": 0.8164722350932145, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_college_medicine": { "acc": 0.2023121387283237, - "acc_stderr": 0.030631145539198823, - "timestamp": "2024-11-21T16-50-41.823534" + "acc_stderr": 0.03063114553919882, + "brier_score": 0.831745806610858, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_business_ethics": { - "acc": 0.35, - "acc_stderr": 0.047937248544110196, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "brier_score": 0.7825093139684296, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_clinical_knowledge": { - "acc": 0.2339622641509434, - "acc_stderr": 0.02605529690115292, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.20754716981132076, + "acc_stderr": 0.02495991802891127, + "brier_score": 0.8186797089881754, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_global_facts": { - "acc": 0.2, - "acc_stderr": 0.04020151261036844, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.19, + "acc_stderr": 0.039427724440366234, + "brier_score": 0.797798978773303, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_virology": { - "acc": 0.2710843373493976, - "acc_stderr": 0.034605799075530276, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.26506024096385544, + "acc_stderr": 0.03436024037944967, + "brier_score": 0.7869259397242031, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_professional_accounting": { - "acc": 0.25886524822695034, - "acc_stderr": 0.026129572527180848, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.24113475177304963, + "acc_stderr": 0.025518731049537766, + "brier_score": 0.8298314768200078, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_college_physics": { - "acc": 0.16666666666666666, - "acc_stderr": 0.03708284662416545, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.21568627450980393, + "acc_stderr": 0.040925639582376556, + "brier_score": 0.8203894402472959, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_high_school_physics": { - "acc": 0.2781456953642384, - "acc_stderr": 0.03658603262763743, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.23178807947019867, + "acc_stderr": 0.03445406271987053, + "brier_score": 0.8125147376482349, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_high_school_biology": { - "acc": 0.20967741935483872, - "acc_stderr": 0.02315787934908353, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.1967741935483871, + "acc_stderr": 0.02261640942074203, + "brier_score": 0.8417325335916167, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_college_biology": { - "acc": 0.25, - "acc_stderr": 0.03621034121889507, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.2569444444444444, + "acc_stderr": 0.03653946969442099, + "brier_score": 0.8147040991164677, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_anatomy": { - "acc": 0.1925925925925926, - "acc_stderr": 0.034065420585026526, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.17777777777777778, + "acc_stderr": 0.03302789859901718, + "brier_score": 0.8490402739177968, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_college_chemistry": { "acc": 0.24, "acc_stderr": 0.042923469599092816, - "timestamp": "2024-11-21T16-50-41.823534" + "brier_score": 0.8065386643467295, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_computer_security": { "acc": 0.28, "acc_stderr": 0.045126085985421276, - "timestamp": "2024-11-21T16-50-41.823534" + "brier_score": 0.8036688727566377, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_college_computer_science": { - "acc": 0.32, - "acc_stderr": 0.04688261722621504, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.24, + "acc_stderr": 0.04292346959909283, + "brier_score": 0.8542327416264095, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_astronomy": { "acc": 0.19736842105263158, "acc_stderr": 0.03238981601699397, - "timestamp": "2024-11-21T16-50-41.823534" + "brier_score": 0.8461938805111608, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_college_mathematics": { - "acc": 0.31, - "acc_stderr": 0.04648231987117316, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.22, + "acc_stderr": 0.041633319989322695, + "brier_score": 0.8212076844205903, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_conceptual_physics": { - "acc": 0.30638297872340425, - "acc_stderr": 0.030135906478517563, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.2765957446808511, + "acc_stderr": 0.029241883869628837, + "brier_score": 0.7958415984856299, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_abstract_algebra": { - "acc": 0.26, - "acc_stderr": 0.04408440022768077, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.27, + "acc_stderr": 0.04461960433384741, + "brier_score": 0.8162146327227109, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_high_school_computer_science": { - "acc": 0.25, - "acc_stderr": 0.04351941398892446, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.24, + "acc_stderr": 0.04292346959909282, + "brier_score": 0.8478307013393968, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_machine_learning": { - "acc": 0.29464285714285715, - "acc_stderr": 0.0432704093257873, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.32142857142857145, + "acc_stderr": 0.04432804055291519, + "brier_score": 0.7652253306062621, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_high_school_chemistry": { - "acc": 0.27586206896551724, - "acc_stderr": 0.031447125816782405, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.1724137931034483, + "acc_stderr": 0.02657767218303657, + "brier_score": 0.8502229478591244, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_high_school_statistics": { - "acc": 0.375, - "acc_stderr": 0.033016908987210894, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.14814814814814814, + "acc_stderr": 0.024227629273728353, + "brier_score": 0.8688155406417832, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_elementary_mathematics": { - "acc": 0.2328042328042328, - "acc_stderr": 0.021765961672154537, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.2222222222222222, + "acc_stderr": 0.02141168439369418, + "brier_score": 0.8255646328708173, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_electrical_engineering": { - "acc": 0.20689655172413793, - "acc_stderr": 0.03375672449560553, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.2413793103448276, + "acc_stderr": 0.03565998174135302, + "brier_score": 0.8134734193210041, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "mmlu_high_school_mathematics": { - "acc": 0.2777777777777778, - "acc_stderr": 0.027309140588230172, - "timestamp": "2024-11-21T16-50-41.823534" + "acc": 0.24814814814814815, + "acc_stderr": 0.026335739404055803, + "brier_score": 0.8201107377136791, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-12-28.954906" }, "arc_challenge": { "acc": 0.21416382252559726, diff --git a/metadata/model_scores/results_Salesforce__codegen-6B-multi.json b/metadata/model_scores/results_Salesforce__codegen-6B-multi.json index 1ace13f..d1ee4fa 100644 --- a/metadata/model_scores/results_Salesforce__codegen-6B-multi.json +++ b/metadata/model_scores/results_Salesforce__codegen-6B-multi.json @@ -1,6 +1,6 @@ { "model_name": "Salesforce/codegen-6B-multi", - "last_updated": "2024-12-04 11:25:55.658914", + "last_updated": "2024-12-19 13:42:14.316186", "results": { "harness": { "minerva_math_precalc": { @@ -228,289 +228,403 @@ "timestamp": "2024-06-13T23-55-47.686622" }, "mmlu_world_religions": { - "acc": 0.2807017543859649, - "acc_stderr": 0.03446296217088427, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.28654970760233917, + "acc_stderr": 0.03467826685703826, + "brier_score": 0.7532546503632859, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_formal_logic": { - "acc": 0.2222222222222222, - "acc_stderr": 0.03718489006818114, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.23015873015873015, + "acc_stderr": 0.03764950879790605, + "brier_score": 0.7963662753131191, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_prehistory": { - "acc": 0.23765432098765432, - "acc_stderr": 0.02368359183700856, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.30864197530864196, + "acc_stderr": 0.025702640260603756, + "brier_score": 0.7634272189938609, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_moral_scenarios": { "acc": 0.2435754189944134, "acc_stderr": 0.01435591196476786, - "timestamp": "2024-11-20T17-44-14.058126" + "brier_score": 0.7995356694042249, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_high_school_world_history": { - "acc": 0.270042194092827, - "acc_stderr": 0.028900721906293426, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.25316455696202533, + "acc_stderr": 0.02830465794303529, + "brier_score": 0.7783172821292649, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_moral_disputes": { - "acc": 0.2861271676300578, - "acc_stderr": 0.02433214677913413, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.26011560693641617, + "acc_stderr": 0.023618678310069356, + "brier_score": 0.7578899242156162, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_professional_law": { - "acc": 0.258148631029987, - "acc_stderr": 0.011176923719313394, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.27249022164276404, + "acc_stderr": 0.01137165829431152, + "brier_score": 0.7811283194404477, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_logical_fallacies": { - "acc": 0.27607361963190186, - "acc_stderr": 0.03512385283705051, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.24539877300613497, + "acc_stderr": 0.03380939813943354, + "brier_score": 0.7622586240232632, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_high_school_us_history": { - "acc": 0.25, - "acc_stderr": 0.03039153369274154, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.25980392156862747, + "acc_stderr": 0.030778554678693254, + "brier_score": 0.7733944586470429, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_philosophy": { - "acc": 0.3022508038585209, - "acc_stderr": 0.02608270069539966, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.24437299035369775, + "acc_stderr": 0.024406162094668903, + "brier_score": 0.7801192892080534, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_jurisprudence": { - "acc": 0.23148148148148148, - "acc_stderr": 0.04077494709252627, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.24074074074074073, + "acc_stderr": 0.041331194402438376, + "brier_score": 0.7768030237415023, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_international_law": { - "acc": 0.3305785123966942, - "acc_stderr": 0.04294340845212095, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.2231404958677686, + "acc_stderr": 0.03800754475228733, + "brier_score": 0.7713861591060401, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_high_school_european_history": { - "acc": 0.2787878787878788, - "acc_stderr": 0.03501438706296781, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.21818181818181817, + "acc_stderr": 0.03225078108306289, + "brier_score": 0.7924488274974129, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_high_school_government_and_politics": { - "acc": 0.20725388601036268, - "acc_stderr": 0.029252823291803627, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.23834196891191708, + "acc_stderr": 0.03074890536390989, + "brier_score": 0.7750805919996284, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_high_school_microeconomics": { - "acc": 0.25630252100840334, - "acc_stderr": 0.02835962087053395, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.20168067226890757, + "acc_stderr": 0.026064313406304516, + "brier_score": 0.791471356035099, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_high_school_geography": { - "acc": 0.20707070707070707, - "acc_stderr": 0.028869778460267063, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.25252525252525254, + "acc_stderr": 0.030954055470365928, + "brier_score": 0.7784681107970716, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_high_school_psychology": { - "acc": 0.25321100917431194, - "acc_stderr": 0.018644073041375046, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.23853211009174313, + "acc_stderr": 0.018272575810231867, + "brier_score": 0.8000259036283063, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_public_relations": { - "acc": 0.3181818181818182, - "acc_stderr": 0.04461272175910509, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.2909090909090909, + "acc_stderr": 0.04350271442923243, + "brier_score": 0.7603960425758571, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_us_foreign_policy": { - "acc": 0.18, - "acc_stderr": 0.03861229196653697, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 0.745432066974931, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_sociology": { - "acc": 0.23880597014925373, - "acc_stderr": 0.030147775935409217, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.31343283582089554, + "acc_stderr": 0.032801882053486435, + "brier_score": 0.7595897474419355, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_high_school_macroeconomics": { - "acc": 0.2205128205128205, - "acc_stderr": 0.021020672680827912, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.21794871794871795, + "acc_stderr": 0.02093244577446319, + "brier_score": 0.7868715882088531, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_security_studies": { - "acc": 0.17551020408163265, - "acc_stderr": 0.02435280072297001, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.22040816326530613, + "acc_stderr": 0.026537045312145312, + "brier_score": 0.7739147450326144, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_professional_psychology": { - "acc": 0.2565359477124183, - "acc_stderr": 0.017667841612378988, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.28594771241830064, + "acc_stderr": 0.01828048507295467, + "brier_score": 0.7681059134757757, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_human_sexuality": { - "acc": 0.1984732824427481, - "acc_stderr": 0.034981493854624714, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.2366412213740458, + "acc_stderr": 0.03727673575596917, + "brier_score": 0.7789988875767444, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_econometrics": { - "acc": 0.2719298245614035, - "acc_stderr": 0.04185774424022056, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.2543859649122807, + "acc_stderr": 0.040969851398436716, + "brier_score": 0.7831895284106645, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_miscellaneous": { - "acc": 0.28607918263090676, - "acc_stderr": 0.01616087140512753, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.26053639846743293, + "acc_stderr": 0.015696008563807092, + "brier_score": 0.7788589895933121, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_marketing": { - "acc": 0.2264957264957265, - "acc_stderr": 0.02742100729539292, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.24786324786324787, + "acc_stderr": 0.028286324075564386, + "brier_score": 0.7764270748713787, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_management": { - "acc": 0.17475728155339806, - "acc_stderr": 0.0376017800602662, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.20388349514563106, + "acc_stderr": 0.03989139859531771, + "brier_score": 0.8192650638978154, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_nutrition": { - "acc": 0.26143790849673204, - "acc_stderr": 0.025160998214292456, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.23529411764705882, + "acc_stderr": 0.024288619466046102, + "brier_score": 0.7963416968181365, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_medical_genetics": { - "acc": 0.26, - "acc_stderr": 0.04408440022768078, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.32, + "acc_stderr": 0.046882617226215034, + "brier_score": 0.7655906441278336, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_human_aging": { - "acc": 0.3901345291479821, - "acc_stderr": 0.03273766725459157, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.34080717488789236, + "acc_stderr": 0.03181149747055362, + "brier_score": 0.7586127872502773, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_professional_medicine": { - "acc": 0.19117647058823528, - "acc_stderr": 0.023886881922440328, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.18382352941176472, + "acc_stderr": 0.023529242185193106, + "brier_score": 0.8281083010413719, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_college_medicine": { - "acc": 0.24277456647398843, - "acc_stderr": 0.0326926380614177, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.18497109826589594, + "acc_stderr": 0.029605623981771214, + "brier_score": 0.7884521080102289, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_business_ethics": { - "acc": 0.26, - "acc_stderr": 0.044084400227680794, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.27, + "acc_stderr": 0.04461960433384739, + "brier_score": 0.7601396063483138, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_clinical_knowledge": { - "acc": 0.27169811320754716, - "acc_stderr": 0.027377706624670713, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.24528301886792453, + "acc_stderr": 0.026480357179895688, + "brier_score": 0.7768285415707459, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_global_facts": { - "acc": 0.24, - "acc_stderr": 0.042923469599092816, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.26, + "acc_stderr": 0.04408440022768078, + "brier_score": 0.7540050020585366, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_virology": { - "acc": 0.3132530120481928, - "acc_stderr": 0.036108050180310235, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.27710843373493976, + "acc_stderr": 0.03484331592680587, + "brier_score": 0.7571894187167946, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_professional_accounting": { - "acc": 0.22695035460992907, - "acc_stderr": 0.024987106365642973, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.28368794326241137, + "acc_stderr": 0.02689170942834396, + "brier_score": 0.7648754774992964, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_college_physics": { - "acc": 0.19607843137254902, - "acc_stderr": 0.0395058186117996, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.28431372549019607, + "acc_stderr": 0.04488482852329017, + "brier_score": 0.7936455868848244, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_high_school_physics": { - "acc": 0.2847682119205298, - "acc_stderr": 0.03684881521389024, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.23841059602649006, + "acc_stderr": 0.0347918557259966, + "brier_score": 0.8213075228649029, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_high_school_biology": { - "acc": 0.23548387096774193, - "acc_stderr": 0.024137632429337717, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.25161290322580643, + "acc_stderr": 0.024685979286239966, + "brier_score": 0.7738057216716354, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_college_biology": { - "acc": 0.2847222222222222, - "acc_stderr": 0.03773809990686934, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.2569444444444444, + "acc_stderr": 0.03653946969442099, + "brier_score": 0.7558366567479475, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_anatomy": { - "acc": 0.28888888888888886, - "acc_stderr": 0.0391545063041425, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.25925925925925924, + "acc_stderr": 0.03785714465066653, + "brier_score": 0.7646735335299525, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_college_chemistry": { - "acc": 0.2, - "acc_stderr": 0.040201512610368445, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.19, + "acc_stderr": 0.039427724440366234, + "brier_score": 0.8108890731790384, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_computer_security": { - "acc": 0.3, - "acc_stderr": 0.046056618647183814, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.22, + "acc_stderr": 0.04163331998932269, + "brier_score": 0.8018378565338942, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_college_computer_science": { - "acc": 0.25, - "acc_stderr": 0.04351941398892446, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.21, + "acc_stderr": 0.04093601807403326, + "brier_score": 0.8188608547002265, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_astronomy": { - "acc": 0.21710526315789475, - "acc_stderr": 0.03355045304882924, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.20394736842105263, + "acc_stderr": 0.0327900040631005, + "brier_score": 0.806676012631122, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_college_mathematics": { - "acc": 0.23, - "acc_stderr": 0.04229525846816506, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.24, + "acc_stderr": 0.04292346959909284, + "brier_score": 0.8343213256984789, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_conceptual_physics": { - "acc": 0.3446808510638298, - "acc_stderr": 0.031068985963122145, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.2680851063829787, + "acc_stderr": 0.028957342788342347, + "brier_score": 0.767174864730072, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_abstract_algebra": { - "acc": 0.31, - "acc_stderr": 0.04648231987117316, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.28, + "acc_stderr": 0.04512608598542127, + "brier_score": 0.8067165829480192, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_high_school_computer_science": { "acc": 0.31, "acc_stderr": 0.04648231987117316, - "timestamp": "2024-11-20T17-44-14.058126" + "brier_score": 0.7509139065340114, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_machine_learning": { - "acc": 0.2857142857142857, - "acc_stderr": 0.04287858751340456, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.29464285714285715, + "acc_stderr": 0.0432704093257873, + "brier_score": 0.769449635318899, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_high_school_chemistry": { - "acc": 0.2512315270935961, - "acc_stderr": 0.030516530732694436, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.2955665024630542, + "acc_stderr": 0.032104944337514575, + "brier_score": 0.7649732533889056, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_high_school_statistics": { - "acc": 0.26851851851851855, - "acc_stderr": 0.030225226160012404, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.19444444444444445, + "acc_stderr": 0.026991454502036726, + "brier_score": 0.8146233490492436, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_elementary_mathematics": { - "acc": 0.24074074074074073, - "acc_stderr": 0.02201908001221789, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.24603174603174602, + "acc_stderr": 0.022182037202948365, + "brier_score": 0.824729102982892, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_electrical_engineering": { - "acc": 0.3103448275862069, - "acc_stderr": 0.03855289616378948, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.30344827586206896, + "acc_stderr": 0.038312260488503336, + "brier_score": 0.7487325148507185, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "mmlu_high_school_mathematics": { - "acc": 0.24444444444444444, - "acc_stderr": 0.026202766534652148, - "timestamp": "2024-11-20T17-44-14.058126" + "acc": 0.2037037037037037, + "acc_stderr": 0.02455617221914127, + "brier_score": 0.8460906774646326, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-18-19.747795" }, "arc_challenge": { "acc": 0.22440273037542663, diff --git a/metadata/model_scores/results_aisingapore__sea-lion-7b.json b/metadata/model_scores/results_aisingapore__sea-lion-7b.json index 8aa0e88..4abdb46 100644 --- a/metadata/model_scores/results_aisingapore__sea-lion-7b.json +++ b/metadata/model_scores/results_aisingapore__sea-lion-7b.json @@ -1,6 +1,6 @@ { "model_name": "aisingapore/sea-lion-7b", - "last_updated": "2024-12-04 11:24:01.901713", + "last_updated": "2024-12-19 13:39:27.519814", "results": { "harness": { "minerva_math_precalc": { @@ -228,289 +228,403 @@ "timestamp": "2024-06-12T10-07-23.284318" }, "mmlu_world_religions": { - "acc": 0.30994152046783624, - "acc_stderr": 0.03546976959393164, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.22807017543859648, + "acc_stderr": 0.03218093795602357, + "brier_score": 0.7752774085491143, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_formal_logic": { - "acc": 0.2619047619047619, - "acc_stderr": 0.039325376803928724, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.30158730158730157, + "acc_stderr": 0.04104947269903394, + "brier_score": 0.7502519936902966, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_prehistory": { - "acc": 0.25, - "acc_stderr": 0.02409347123262133, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.3055555555555556, + "acc_stderr": 0.025630824975621344, + "brier_score": 0.7520781517274742, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_moral_scenarios": { - "acc": 0.27262569832402234, - "acc_stderr": 0.014893391735249603, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.2435754189944134, + "acc_stderr": 0.01435591196476786, + "brier_score": 0.757214810611251, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_high_school_world_history": { - "acc": 0.28270042194092826, - "acc_stderr": 0.029312814153955917, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.2616033755274262, + "acc_stderr": 0.028609516716994934, + "brier_score": 0.7731902924413044, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_moral_disputes": { - "acc": 0.2976878612716763, - "acc_stderr": 0.024617055388677003, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.2745664739884393, + "acc_stderr": 0.024027745155265012, + "brier_score": 0.7576887106989154, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_professional_law": { - "acc": 0.288135593220339, - "acc_stderr": 0.011567140661324568, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.2607561929595828, + "acc_stderr": 0.01121347155960232, + "brier_score": 0.7605541627446022, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_logical_fallacies": { - "acc": 0.25153374233128833, - "acc_stderr": 0.034089978868575295, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.31901840490797545, + "acc_stderr": 0.03661997551073836, + "brier_score": 0.7547284277689329, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_high_school_us_history": { - "acc": 0.25980392156862747, - "acc_stderr": 0.03077855467869326, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.2549019607843137, + "acc_stderr": 0.03058759135160425, + "brier_score": 0.7724803077641238, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_philosophy": { - "acc": 0.3054662379421222, - "acc_stderr": 0.026160584450140485, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.3215434083601286, + "acc_stderr": 0.026527724079528872, + "brier_score": 0.738986747794473, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_jurisprudence": { - "acc": 0.28703703703703703, - "acc_stderr": 0.043733130409147614, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.3425925925925926, + "acc_stderr": 0.04587904741301812, + "brier_score": 0.7502170737990164, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_international_law": { - "acc": 0.21487603305785125, - "acc_stderr": 0.037494924487096966, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.30578512396694213, + "acc_stderr": 0.04205953933884124, + "brier_score": 0.7544421936349635, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_high_school_european_history": { - "acc": 0.26666666666666666, - "acc_stderr": 0.03453131801885415, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.296969696969697, + "acc_stderr": 0.03567969772268049, + "brier_score": 0.7638611557055202, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_high_school_government_and_politics": { - "acc": 0.29015544041450775, - "acc_stderr": 0.03275264467791515, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.22279792746113988, + "acc_stderr": 0.03003114797764154, + "brier_score": 0.7592438626250126, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_high_school_microeconomics": { - "acc": 0.23109243697478993, - "acc_stderr": 0.027381406927868973, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.226890756302521, + "acc_stderr": 0.027205371538279483, + "brier_score": 0.7520425312177341, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_high_school_geography": { - "acc": 0.35353535353535354, - "acc_stderr": 0.03406086723547153, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.23232323232323232, + "acc_stderr": 0.030088629490217483, + "brier_score": 0.7559098136708166, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_high_school_psychology": { - "acc": 0.27339449541284405, - "acc_stderr": 0.019109299846098278, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.23853211009174313, + "acc_stderr": 0.01827257581023186, + "brier_score": 0.7614810618696795, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_public_relations": { - "acc": 0.2818181818181818, - "acc_stderr": 0.043091187099464606, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.23636363636363636, + "acc_stderr": 0.04069306319721376, + "brier_score": 0.7607146453285218, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_us_foreign_policy": { - "acc": 0.31, - "acc_stderr": 0.04648231987117316, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.33, + "acc_stderr": 0.04725815626252606, + "brier_score": 0.7384479897630887, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_sociology": { - "acc": 0.2885572139303483, - "acc_stderr": 0.03203841040213322, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.25870646766169153, + "acc_stderr": 0.030965903123573037, + "brier_score": 0.7578779619880223, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_high_school_macroeconomics": { - "acc": 0.2923076923076923, - "acc_stderr": 0.023060438380857747, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.24358974358974358, + "acc_stderr": 0.02176373368417392, + "brier_score": 0.7472250881282662, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_security_studies": { - "acc": 0.44081632653061226, - "acc_stderr": 0.03178419114175363, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.4, + "acc_stderr": 0.03136250240935893, + "brier_score": 0.7162989736314286, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_professional_psychology": { - "acc": 0.2630718954248366, - "acc_stderr": 0.017812676542320657, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.27124183006535946, + "acc_stderr": 0.017986615304030312, + "brier_score": 0.7546982787291897, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_human_sexuality": { - "acc": 0.2824427480916031, - "acc_stderr": 0.03948406125768361, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.31297709923664124, + "acc_stderr": 0.04066962905677698, + "brier_score": 0.7554849086865097, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_econometrics": { - "acc": 0.2807017543859649, - "acc_stderr": 0.042270544512321984, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.2543859649122807, + "acc_stderr": 0.0409698513984367, + "brier_score": 0.7646334716879251, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_miscellaneous": { - "acc": 0.26053639846743293, - "acc_stderr": 0.01569600856380709, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.26309067688378035, + "acc_stderr": 0.015745497169049046, + "brier_score": 0.7643217133064008, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_marketing": { - "acc": 0.2606837606837607, - "acc_stderr": 0.028760348956523414, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.23504273504273504, + "acc_stderr": 0.027778835904935423, + "brier_score": 0.7756730657667739, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_management": { - "acc": 0.23300970873786409, - "acc_stderr": 0.04185832598928315, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.24271844660194175, + "acc_stderr": 0.04245022486384493, + "brier_score": 0.7511358086600124, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_nutrition": { - "acc": 0.28104575163398693, - "acc_stderr": 0.02573885479781871, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.3104575163398693, + "acc_stderr": 0.0264930332251459, + "brier_score": 0.7471881741064178, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_medical_genetics": { - "acc": 0.23, - "acc_stderr": 0.04229525846816506, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.24, + "acc_stderr": 0.042923469599092816, + "brier_score": 0.7655001909042981, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_human_aging": { - "acc": 0.20179372197309417, - "acc_stderr": 0.026936111912802256, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.26905829596412556, + "acc_stderr": 0.029763779406874975, + "brier_score": 0.7839268773478784, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_professional_medicine": { - "acc": 0.19852941176470587, - "acc_stderr": 0.024231013370541073, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.20220588235294118, + "acc_stderr": 0.024398192986654924, + "brier_score": 0.7528840585295138, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_college_medicine": { - "acc": 0.3583815028901734, - "acc_stderr": 0.03656343653353159, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.2543352601156069, + "acc_stderr": 0.0332055644308557, + "brier_score": 0.7539792708412846, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_business_ethics": { - "acc": 0.31, - "acc_stderr": 0.04648231987117316, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.16, + "acc_stderr": 0.036845294917747094, + "brier_score": 0.7729585342994129, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_clinical_knowledge": { - "acc": 0.2943396226415094, - "acc_stderr": 0.028049186315695248, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.27169811320754716, + "acc_stderr": 0.02737770662467071, + "brier_score": 0.7556143704191626, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_global_facts": { - "acc": 0.29, - "acc_stderr": 0.045604802157206845, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.33, + "acc_stderr": 0.04725815626252605, + "brier_score": 0.7480811312516856, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_virology": { - "acc": 0.25903614457831325, - "acc_stderr": 0.034106466140718564, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.29518072289156627, + "acc_stderr": 0.0355092018568963, + "brier_score": 0.7591723780375526, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_professional_accounting": { - "acc": 0.2695035460992908, - "acc_stderr": 0.026469036818590634, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.25886524822695034, + "acc_stderr": 0.026129572527180848, + "brier_score": 0.7633880028569879, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_college_physics": { - "acc": 0.1568627450980392, - "acc_stderr": 0.036186648199362466, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.27450980392156865, + "acc_stderr": 0.044405219061793254, + "brier_score": 0.7585494145393273, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_high_school_physics": { - "acc": 0.23178807947019867, - "acc_stderr": 0.034454062719870546, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.32450331125827814, + "acc_stderr": 0.03822746937658753, + "brier_score": 0.7394946142667588, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_high_school_biology": { - "acc": 0.23225806451612904, - "acc_stderr": 0.02402225613030824, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.27741935483870966, + "acc_stderr": 0.025470196835900055, + "brier_score": 0.7485462690398437, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_college_biology": { - "acc": 0.3263888888888889, - "acc_stderr": 0.03921067198982266, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.25, + "acc_stderr": 0.03621034121889507, + "brier_score": 0.763165184784987, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_anatomy": { - "acc": 0.22962962962962963, - "acc_stderr": 0.03633384414073465, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.3111111111111111, + "acc_stderr": 0.03999262876617724, + "brier_score": 0.734966060147871, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_college_chemistry": { - "acc": 0.21, - "acc_stderr": 0.040936018074033256, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.22, + "acc_stderr": 0.04163331998932269, + "brier_score": 0.7627976595239333, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_computer_security": { - "acc": 0.31, - "acc_stderr": 0.046482319871173156, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "brier_score": 0.7738218912003072, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_college_computer_science": { "acc": 0.31, "acc_stderr": 0.04648231987117316, - "timestamp": "2024-11-21T21-51-41.609919" + "brier_score": 0.747236755807785, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_astronomy": { - "acc": 0.26973684210526316, - "acc_stderr": 0.03611780560284898, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.3223684210526316, + "acc_stderr": 0.038035102483515854, + "brier_score": 0.7375677113730162, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_college_mathematics": { - "acc": 0.26, - "acc_stderr": 0.0440844002276808, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.22, + "acc_stderr": 0.04163331998932269, + "brier_score": 0.756023191070122, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_conceptual_physics": { - "acc": 0.23829787234042554, - "acc_stderr": 0.027851252973889795, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.225531914893617, + "acc_stderr": 0.027321078417387533, + "brier_score": 0.7741473196300603, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_abstract_algebra": { - "acc": 0.25, - "acc_stderr": 0.04351941398892446, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.24, + "acc_stderr": 0.04292346959909284, + "brier_score": 0.7697823399701942, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_high_school_computer_science": { - "acc": 0.28, - "acc_stderr": 0.04512608598542127, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "brier_score": 0.750020760483219, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_machine_learning": { - "acc": 0.25892857142857145, - "acc_stderr": 0.04157751539865629, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.29464285714285715, + "acc_stderr": 0.04327040932578728, + "brier_score": 0.7839518900665153, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_high_school_chemistry": { - "acc": 0.22660098522167488, - "acc_stderr": 0.02945486383529298, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.32019704433497537, + "acc_stderr": 0.032826493853041504, + "brier_score": 0.7354908609195902, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_high_school_statistics": { - "acc": 0.25462962962962965, - "acc_stderr": 0.029711275860005357, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.2175925925925926, + "acc_stderr": 0.02813968944485967, + "brier_score": 0.7414971120687751, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_elementary_mathematics": { - "acc": 0.25132275132275134, - "acc_stderr": 0.022340482339643895, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.28835978835978837, + "acc_stderr": 0.023330654054535882, + "brier_score": 0.7529959049377857, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_electrical_engineering": { - "acc": 0.31724137931034485, - "acc_stderr": 0.03878352372138622, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.3103448275862069, + "acc_stderr": 0.03855289616378949, + "brier_score": 0.7518379881767014, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "mmlu_high_school_mathematics": { - "acc": 0.22592592592592592, - "acc_stderr": 0.02549753263960955, - "timestamp": "2024-11-21T21-51-41.609919" + "acc": 0.2740740740740741, + "acc_stderr": 0.027195934804085622, + "brier_score": 0.7608680277559734, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-58-13.342162" }, "arc_challenge": { "acc": 0.35921501706484643, diff --git a/metadata/model_scores/results_bigscience__bloom-1b7.json b/metadata/model_scores/results_bigscience__bloom-1b7.json index b308f52..956df24 100644 --- a/metadata/model_scores/results_bigscience__bloom-1b7.json +++ b/metadata/model_scores/results_bigscience__bloom-1b7.json @@ -1,6 +1,6 @@ { "model_name": "bigscience/bloom-1b7", - "last_updated": "2024-12-04 11:22:54.614903", + "last_updated": "2024-12-19 13:38:14.414566", "results": { "harness": { "minerva_math_precalc": { @@ -228,289 +228,403 @@ "timestamp": "2024-06-14T17-01-04.888098" }, "mmlu_world_religions": { - "acc": 0.26900584795321636, - "acc_stderr": 0.034010526201040905, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.2982456140350877, + "acc_stderr": 0.03508771929824565, + "brier_score": 0.7542430156573607, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_formal_logic": { - "acc": 0.35714285714285715, - "acc_stderr": 0.04285714285714281, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.14285714285714285, + "acc_stderr": 0.031298431857438094, + "brier_score": 0.798740683570427, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_prehistory": { - "acc": 0.22530864197530864, - "acc_stderr": 0.023246202647819746, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.2839506172839506, + "acc_stderr": 0.02508947852376513, + "brier_score": 0.7590573428880001, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_moral_scenarios": { - "acc": 0.27262569832402234, - "acc_stderr": 0.014893391735249603, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.23687150837988827, + "acc_stderr": 0.014219570788103987, + "brier_score": 0.760565800135267, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_high_school_world_history": { - "acc": 0.24472573839662448, - "acc_stderr": 0.02798569938703643, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.26582278481012656, + "acc_stderr": 0.028756799629658346, + "brier_score": 0.7655519522477238, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_moral_disputes": { - "acc": 0.27167630057803466, - "acc_stderr": 0.023948512905468365, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.30057803468208094, + "acc_stderr": 0.024685316867257803, + "brier_score": 0.7566306701713452, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_professional_law": { - "acc": 0.2646675358539765, - "acc_stderr": 0.011267332992845533, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.2653194263363755, + "acc_stderr": 0.011276198843958871, + "brier_score": 0.763543464923393, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_logical_fallacies": { - "acc": 0.26993865030674846, - "acc_stderr": 0.034878251684978906, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.2883435582822086, + "acc_stderr": 0.035590395316173425, + "brier_score": 0.7541555610476428, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_high_school_us_history": { - "acc": 0.24509803921568626, - "acc_stderr": 0.03019028245350194, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.25, + "acc_stderr": 0.03039153369274154, + "brier_score": 0.7769130551084145, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_philosophy": { - "acc": 0.2797427652733119, - "acc_stderr": 0.025494259350694902, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.2508038585209003, + "acc_stderr": 0.02461977195669716, + "brier_score": 0.781006843058891, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_jurisprudence": { - "acc": 0.24074074074074073, - "acc_stderr": 0.04133119440243838, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.25925925925925924, + "acc_stderr": 0.04236511258094633, + "brier_score": 0.7708865794629522, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_international_law": { - "acc": 0.2231404958677686, - "acc_stderr": 0.03800754475228733, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.35537190082644626, + "acc_stderr": 0.0436923632657398, + "brier_score": 0.7234428693278497, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_high_school_european_history": { - "acc": 0.2787878787878788, - "acc_stderr": 0.03501438706296781, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.26666666666666666, + "acc_stderr": 0.03453131801885415, + "brier_score": 0.7755857983182882, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_high_school_government_and_politics": { - "acc": 0.37823834196891193, - "acc_stderr": 0.03499807276193338, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.24352331606217617, + "acc_stderr": 0.030975436386845436, + "brier_score": 0.7869607311121265, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_high_school_microeconomics": { - "acc": 0.29411764705882354, - "acc_stderr": 0.02959732973097809, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.21428571428571427, + "acc_stderr": 0.026653531596715473, + "brier_score": 0.792844202748864, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_high_school_geography": { - "acc": 0.36363636363636365, - "acc_stderr": 0.03427308652999936, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.24242424242424243, + "acc_stderr": 0.030532892233932036, + "brier_score": 0.758184696530688, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_high_school_psychology": { - "acc": 0.3486238532110092, - "acc_stderr": 0.020431254090714328, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.22752293577981653, + "acc_stderr": 0.017974463578776502, + "brier_score": 0.7763696246652465, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_public_relations": { - "acc": 0.2727272727272727, - "acc_stderr": 0.04265792110940589, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.21818181818181817, + "acc_stderr": 0.03955932861795833, + "brier_score": 0.7678053749290249, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_us_foreign_policy": { - "acc": 0.26, - "acc_stderr": 0.0440844002276808, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "brier_score": 0.7840269984992296, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_sociology": { - "acc": 0.26865671641791045, - "acc_stderr": 0.03134328358208954, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.263681592039801, + "acc_stderr": 0.031157150869355568, + "brier_score": 0.7852952947920849, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_high_school_macroeconomics": { - "acc": 0.358974358974359, - "acc_stderr": 0.024321738484602364, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.22564102564102564, + "acc_stderr": 0.02119363252514854, + "brier_score": 0.7935731623921292, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_security_studies": { - "acc": 0.4, - "acc_stderr": 0.031362502409358936, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.2, + "acc_stderr": 0.025607375986579153, + "brier_score": 0.8063760510247837, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_professional_psychology": { - "acc": 0.2908496732026144, - "acc_stderr": 0.018373116915903966, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.26633986928104575, + "acc_stderr": 0.017883188134667195, + "brier_score": 0.7657508919797539, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_human_sexuality": { - "acc": 0.22900763358778625, - "acc_stderr": 0.036853466317118506, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.2595419847328244, + "acc_stderr": 0.03844876139785271, + "brier_score": 0.7665013825604922, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_econometrics": { - "acc": 0.23684210526315788, - "acc_stderr": 0.039994238792813344, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.2543859649122807, + "acc_stderr": 0.040969851398436695, + "brier_score": 0.766316971646447, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_miscellaneous": { - "acc": 0.20434227330779056, - "acc_stderr": 0.0144191239809319, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.2784163473818646, + "acc_stderr": 0.016028295188992462, + "brier_score": 0.7594185589263097, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_marketing": { - "acc": 0.23931623931623933, - "acc_stderr": 0.027951826808924333, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.2863247863247863, + "acc_stderr": 0.02961432369045665, + "brier_score": 0.7569178236298468, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_management": { - "acc": 0.3883495145631068, - "acc_stderr": 0.0482572933735639, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.24271844660194175, + "acc_stderr": 0.042450224863844935, + "brier_score": 0.8133530185364424, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_nutrition": { - "acc": 0.2777777777777778, - "acc_stderr": 0.02564686309713792, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.24836601307189543, + "acc_stderr": 0.02473998135511359, + "brier_score": 0.78442135518889, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_medical_genetics": { - "acc": 0.25, - "acc_stderr": 0.04351941398892446, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.24, + "acc_stderr": 0.042923469599092816, + "brier_score": 0.7523798314128147, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_human_aging": { - "acc": 0.13452914798206278, - "acc_stderr": 0.02290118376157557, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.19282511210762332, + "acc_stderr": 0.026478240960489365, + "brier_score": 0.7772495068884132, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_professional_medicine": { - "acc": 0.4485294117647059, - "acc_stderr": 0.030211479609121593, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.16176470588235295, + "acc_stderr": 0.022368672562886754, + "brier_score": 0.808249444559704, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_college_medicine": { "acc": 0.24277456647398843, "acc_stderr": 0.0326926380614177, - "timestamp": "2024-11-21T08-38-18.699254" + "brier_score": 0.7880406191469156, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_business_ethics": { - "acc": 0.21, - "acc_stderr": 0.040936018074033256, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 0.771045014592912, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_clinical_knowledge": { - "acc": 0.2981132075471698, - "acc_stderr": 0.028152837942493875, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.23773584905660378, + "acc_stderr": 0.02619980880756193, + "brier_score": 0.7870920404089861, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_global_facts": { - "acc": 0.31, - "acc_stderr": 0.04648231987117316, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.33, + "acc_stderr": 0.047258156262526045, + "brier_score": 0.7574532104546047, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_virology": { - "acc": 0.19879518072289157, - "acc_stderr": 0.031069390260789413, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.25903614457831325, + "acc_stderr": 0.03410646614071856, + "brier_score": 0.7710796189243675, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_professional_accounting": { - "acc": 0.2730496453900709, - "acc_stderr": 0.026577860943307854, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.24468085106382978, + "acc_stderr": 0.02564555362226673, + "brier_score": 0.7762138964007215, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_college_physics": { - "acc": 0.2647058823529412, - "acc_stderr": 0.043898699568087764, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.22549019607843138, + "acc_stderr": 0.041583075330832865, + "brier_score": 0.8187186665326814, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_high_school_physics": { - "acc": 0.33112582781456956, - "acc_stderr": 0.038425817186598696, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.2847682119205298, + "acc_stderr": 0.03684881521389023, + "brier_score": 0.7560166622001294, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_high_school_biology": { - "acc": 0.2967741935483871, - "acc_stderr": 0.0259885007924119, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.23870967741935484, + "acc_stderr": 0.024251071262208834, + "brier_score": 0.7669627733915457, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_college_biology": { - "acc": 0.2777777777777778, - "acc_stderr": 0.03745554791462457, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.2569444444444444, + "acc_stderr": 0.03653946969442099, + "brier_score": 0.76589617503354, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_anatomy": { - "acc": 0.24444444444444444, - "acc_stderr": 0.03712537833614865, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.3333333333333333, + "acc_stderr": 0.04072314811876837, + "brier_score": 0.7545449490443557, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_college_chemistry": { - "acc": 0.22, - "acc_stderr": 0.04163331998932269, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.2, + "acc_stderr": 0.04020151261036845, + "brier_score": 0.7983463688517936, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_computer_security": { - "acc": 0.21, - "acc_stderr": 0.040936018074033256, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.2, + "acc_stderr": 0.04020151261036845, + "brier_score": 0.7738297058247153, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_college_computer_science": { - "acc": 0.33, - "acc_stderr": 0.047258156262526045, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.32, + "acc_stderr": 0.04688261722621503, + "brier_score": 0.7458884425655282, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_astronomy": { - "acc": 0.2631578947368421, - "acc_stderr": 0.03583496176361062, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.29605263157894735, + "acc_stderr": 0.03715062154998905, + "brier_score": 0.752107495455209, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_college_mathematics": { - "acc": 0.31, - "acc_stderr": 0.04648231987117316, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.29, + "acc_stderr": 0.045604802157206845, + "brier_score": 0.7524346393147284, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_conceptual_physics": { - "acc": 0.225531914893617, - "acc_stderr": 0.027321078417387533, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.20425531914893616, + "acc_stderr": 0.02635515841334941, + "brier_score": 0.7891426254381969, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_abstract_algebra": { - "acc": 0.18, - "acc_stderr": 0.03861229196653696, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 0.7721622244111885, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_high_school_computer_science": { - "acc": 0.22, - "acc_stderr": 0.04163331998932269, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.33, + "acc_stderr": 0.04725815626252604, + "brier_score": 0.7375773903853942, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_machine_learning": { - "acc": 0.17857142857142858, - "acc_stderr": 0.036352091215778065, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.1875, + "acc_stderr": 0.0370468111477387, + "brier_score": 0.7739719958530921, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_high_school_chemistry": { - "acc": 0.28078817733990147, - "acc_stderr": 0.0316185633535861, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.30049261083743845, + "acc_stderr": 0.03225799476233486, + "brier_score": 0.7645420958708635, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_high_school_statistics": { - "acc": 0.4722222222222222, - "acc_stderr": 0.0340470532865388, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.27314814814814814, + "acc_stderr": 0.030388051301678116, + "brier_score": 0.7466109237770758, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_elementary_mathematics": { - "acc": 0.2328042328042328, - "acc_stderr": 0.021765961672154523, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.2566137566137566, + "acc_stderr": 0.022494510767503154, + "brier_score": 0.7884536895302897, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_electrical_engineering": { - "acc": 0.2482758620689655, - "acc_stderr": 0.03600105692727772, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.2689655172413793, + "acc_stderr": 0.03695183311650232, + "brier_score": 0.7678659005899077, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "mmlu_high_school_mathematics": { - "acc": 0.25555555555555554, - "acc_stderr": 0.026593939101844058, - "timestamp": "2024-11-21T08-38-18.699254" + "acc": 0.28888888888888886, + "acc_stderr": 0.027634907264178544, + "brier_score": 0.7685854680890092, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-41-35.899405" }, "arc_challenge": { "acc": 0.2627986348122867, diff --git a/metadata/model_scores/results_bigscience__bloom-7b1.json b/metadata/model_scores/results_bigscience__bloom-7b1.json index 130b287..1c4f7b6 100644 --- a/metadata/model_scores/results_bigscience__bloom-7b1.json +++ b/metadata/model_scores/results_bigscience__bloom-7b1.json @@ -1,6 +1,6 @@ { "model_name": "bigscience/bloom-7b1", - "last_updated": "2024-12-04 11:22:57.561775", + "last_updated": "2024-12-19 13:38:18.341415", "results": { "harness": { "minerva_math_precalc": { @@ -228,289 +228,403 @@ "timestamp": "2024-06-14T07-47-06.594201" }, "mmlu_world_religions": { - "acc": 0.29239766081871343, - "acc_stderr": 0.03488647713457922, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.2807017543859649, + "acc_stderr": 0.03446296217088427, + "brier_score": 0.7547868709568236, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_formal_logic": { - "acc": 0.16666666666666666, - "acc_stderr": 0.033333333333333354, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.29365079365079366, + "acc_stderr": 0.040735243221471255, + "brier_score": 0.7542957646039679, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_prehistory": { - "acc": 0.2623456790123457, - "acc_stderr": 0.024477222856135107, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.2191358024691358, + "acc_stderr": 0.023016705640262185, + "brier_score": 0.7523443552886664, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_moral_scenarios": { - "acc": 0.2424581005586592, - "acc_stderr": 0.014333522059217887, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.25139664804469275, + "acc_stderr": 0.014508979453553977, + "brier_score": 0.7510528990492207, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_high_school_world_history": { - "acc": 0.26582278481012656, - "acc_stderr": 0.028756799629658342, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.21518987341772153, + "acc_stderr": 0.02675082699467617, + "brier_score": 0.7615802570185577, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_moral_disputes": { - "acc": 0.24277456647398843, - "acc_stderr": 0.023083658586984204, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.23699421965317918, + "acc_stderr": 0.022894082489925992, + "brier_score": 0.7534295148857957, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_professional_law": { - "acc": 0.2542372881355932, - "acc_stderr": 0.01112112900784068, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.28878748370273793, + "acc_stderr": 0.011574914757219962, + "brier_score": 0.7503956948481256, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_logical_fallacies": { - "acc": 0.26380368098159507, - "acc_stderr": 0.034624199316156234, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.22085889570552147, + "acc_stderr": 0.03259177392742177, + "brier_score": 0.7566267051940508, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_high_school_us_history": { - "acc": 0.27450980392156865, - "acc_stderr": 0.03132179803083292, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.21568627450980393, + "acc_stderr": 0.028867431449849313, + "brier_score": 0.7640591185075365, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_philosophy": { - "acc": 0.2733118971061093, - "acc_stderr": 0.02531176597542612, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.26366559485530544, + "acc_stderr": 0.02502553850053234, + "brier_score": 0.7499865265122302, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_jurisprudence": { - "acc": 0.28703703703703703, - "acc_stderr": 0.043733130409147614, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.1574074074074074, + "acc_stderr": 0.035207039905179635, + "brier_score": 0.764194816756646, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_international_law": { - "acc": 0.35537190082644626, - "acc_stderr": 0.04369236326573981, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.38016528925619836, + "acc_stderr": 0.04431324501968432, + "brier_score": 0.7308189036868278, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_high_school_european_history": { - "acc": 0.23636363636363636, - "acc_stderr": 0.03317505930009179, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.18787878787878787, + "acc_stderr": 0.03050193405942914, + "brier_score": 0.7642456313854691, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_high_school_government_and_politics": { - "acc": 0.20725388601036268, - "acc_stderr": 0.029252823291803627, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.3005181347150259, + "acc_stderr": 0.03308818594415751, + "brier_score": 0.740565281777382, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_high_school_microeconomics": { - "acc": 0.226890756302521, - "acc_stderr": 0.027205371538279472, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.2647058823529412, + "acc_stderr": 0.02865749128507198, + "brier_score": 0.7482446000796693, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_high_school_geography": { - "acc": 0.23232323232323232, - "acc_stderr": 0.030088629490217487, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.3484848484848485, + "acc_stderr": 0.033948539651564025, + "brier_score": 0.7405135376552053, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_high_school_psychology": { - "acc": 0.24403669724770644, - "acc_stderr": 0.018415286351416413, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.29908256880733947, + "acc_stderr": 0.01963041728541517, + "brier_score": 0.7443715431074973, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_public_relations": { - "acc": 0.32727272727272727, - "acc_stderr": 0.04494290866252091, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.2636363636363636, + "acc_stderr": 0.04220224692971987, + "brier_score": 0.7532888263064784, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_us_foreign_policy": { - "acc": 0.21, - "acc_stderr": 0.04093601807403326, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 0.7489981620742795, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_sociology": { - "acc": 0.23880597014925373, - "acc_stderr": 0.030147775935409217, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.24875621890547264, + "acc_stderr": 0.030567675938916718, + "brier_score": 0.7505873310310913, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_high_school_macroeconomics": { - "acc": 0.23076923076923078, - "acc_stderr": 0.021362027725222735, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.31794871794871793, + "acc_stderr": 0.02361088430892786, + "brier_score": 0.7411049977825732, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_security_studies": { - "acc": 0.3142857142857143, - "acc_stderr": 0.02971932942241748, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.2857142857142857, + "acc_stderr": 0.02892058322067561, + "brier_score": 0.7442189649442519, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_professional_psychology": { - "acc": 0.26143790849673204, - "acc_stderr": 0.01777694715752804, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.24836601307189543, + "acc_stderr": 0.017479487001364764, + "brier_score": 0.7563644232783947, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_human_sexuality": { - "acc": 0.2366412213740458, - "acc_stderr": 0.03727673575596918, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.2748091603053435, + "acc_stderr": 0.03915345408847836, + "brier_score": 0.7568870604049708, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_econometrics": { - "acc": 0.24561403508771928, - "acc_stderr": 0.04049339297748142, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.23684210526315788, + "acc_stderr": 0.039994238792813344, + "brier_score": 0.7618061547279509, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_miscellaneous": { - "acc": 0.28991060025542786, - "acc_stderr": 0.01622501794477096, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.24776500638569604, + "acc_stderr": 0.015438083080568963, + "brier_score": 0.7551437519659098, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_marketing": { - "acc": 0.24786324786324787, - "acc_stderr": 0.02828632407556438, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.2863247863247863, + "acc_stderr": 0.02961432369045666, + "brier_score": 0.7505136324296918, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_management": { - "acc": 0.2524271844660194, - "acc_stderr": 0.04301250399690878, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.30097087378640774, + "acc_stderr": 0.0454160944650395, + "brier_score": 0.7481389336784959, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_nutrition": { - "acc": 0.23529411764705882, - "acc_stderr": 0.024288619466046112, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.3104575163398693, + "acc_stderr": 0.02649303322514589, + "brier_score": 0.7431157926009903, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_medical_genetics": { - "acc": 0.26, - "acc_stderr": 0.04408440022768079, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "brier_score": 0.7492520817638646, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_human_aging": { - "acc": 0.3632286995515695, - "acc_stderr": 0.032277904428505, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.19282511210762332, + "acc_stderr": 0.026478240960489365, + "brier_score": 0.7669465589527058, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_professional_medicine": { - "acc": 0.20588235294117646, - "acc_stderr": 0.024562204314142317, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.3014705882352941, + "acc_stderr": 0.027875982114273168, + "brier_score": 0.747362125363743, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_college_medicine": { - "acc": 0.20809248554913296, - "acc_stderr": 0.030952890217749874, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.2658959537572254, + "acc_stderr": 0.03368762932259431, + "brier_score": 0.7503399941846938, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_business_ethics": { "acc": 0.26, - "acc_stderr": 0.0440844002276808, - "timestamp": "2024-11-18T13-06-35.949978" + "acc_stderr": 0.04408440022768081, + "brier_score": 0.7570753162485974, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_clinical_knowledge": { - "acc": 0.27547169811320754, - "acc_stderr": 0.027495663683724064, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.27169811320754716, + "acc_stderr": 0.02737770662467071, + "brier_score": 0.7442302549814358, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_global_facts": { - "acc": 0.32, - "acc_stderr": 0.04688261722621505, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.24, + "acc_stderr": 0.042923469599092816, + "brier_score": 0.7605468395990248, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_virology": { - "acc": 0.3072289156626506, - "acc_stderr": 0.035915667978246635, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.20481927710843373, + "acc_stderr": 0.031417842916639245, + "brier_score": 0.7681776042810181, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_professional_accounting": { - "acc": 0.26595744680851063, - "acc_stderr": 0.026358065698880596, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.24822695035460993, + "acc_stderr": 0.025770015644290382, + "brier_score": 0.7516006983355339, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_college_physics": { - "acc": 0.20588235294117646, - "acc_stderr": 0.04023382273617746, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.22549019607843138, + "acc_stderr": 0.041583075330832865, + "brier_score": 0.7919065334892137, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_high_school_physics": { - "acc": 0.23841059602649006, - "acc_stderr": 0.03479185572599661, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.2980132450331126, + "acc_stderr": 0.03734535676787198, + "brier_score": 0.7490773527348867, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_high_school_biology": { - "acc": 0.25161290322580643, - "acc_stderr": 0.02468597928623997, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.3, + "acc_stderr": 0.026069362295335123, + "brier_score": 0.7506773910715951, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_college_biology": { - "acc": 0.19444444444444445, - "acc_stderr": 0.03309615177059006, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.2916666666666667, + "acc_stderr": 0.03800968060554859, + "brier_score": 0.749773570282803, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_anatomy": { - "acc": 0.24444444444444444, - "acc_stderr": 0.037125378336148665, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.25925925925925924, + "acc_stderr": 0.03785714465066654, + "brier_score": 0.7563532132714177, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_college_chemistry": { - "acc": 0.21, - "acc_stderr": 0.040936018074033256, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.33, + "acc_stderr": 0.047258156262526045, + "brier_score": 0.7495173234527738, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_computer_security": { "acc": 0.23, - "acc_stderr": 0.04229525846816506, - "timestamp": "2024-11-18T13-06-35.949978" + "acc_stderr": 0.04229525846816507, + "brier_score": 0.7597888572881607, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_college_computer_science": { - "acc": 0.33, - "acc_stderr": 0.04725815626252604, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 0.7601643308711965, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_astronomy": { - "acc": 0.17763157894736842, - "acc_stderr": 0.031103182383123387, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.29605263157894735, + "acc_stderr": 0.03715062154998906, + "brier_score": 0.7480539257491657, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_college_mathematics": { - "acc": 0.33, - "acc_stderr": 0.04725815626252606, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.29, + "acc_stderr": 0.04560480215720684, + "brier_score": 0.7361561560209223, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_conceptual_physics": { "acc": 0.32340425531914896, - "acc_stderr": 0.03057944277361035, - "timestamp": "2024-11-18T13-06-35.949978" + "acc_stderr": 0.030579442773610337, + "brier_score": 0.7451312536249421, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_abstract_algebra": { - "acc": 0.26, - "acc_stderr": 0.04408440022768079, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.23, + "acc_stderr": 0.04229525846816506, + "brier_score": 0.7581665860489692, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_high_school_computer_science": { - "acc": 0.33, - "acc_stderr": 0.04725815626252605, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.28, + "acc_stderr": 0.04512608598542127, + "brier_score": 0.7500153040904679, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_machine_learning": { - "acc": 0.30357142857142855, - "acc_stderr": 0.04364226155841044, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.3125, + "acc_stderr": 0.043994650575715215, + "brier_score": 0.7577083286356877, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_high_school_chemistry": { - "acc": 0.270935960591133, - "acc_stderr": 0.031270907132976984, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.24630541871921183, + "acc_stderr": 0.030315099285617722, + "brier_score": 0.7470868809070461, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_high_school_statistics": { - "acc": 0.38425925925925924, - "acc_stderr": 0.03317354514310742, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.26851851851851855, + "acc_stderr": 0.03022522616001239, + "brier_score": 0.7523433337900371, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_elementary_mathematics": { - "acc": 0.2777777777777778, - "acc_stderr": 0.02306818884826112, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.2222222222222222, + "acc_stderr": 0.02141168439369419, + "brier_score": 0.7615287575912638, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_electrical_engineering": { - "acc": 0.2482758620689655, - "acc_stderr": 0.036001056927277716, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.2620689655172414, + "acc_stderr": 0.036646663372252565, + "brier_score": 0.7500765841791096, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "mmlu_high_school_mathematics": { - "acc": 0.25555555555555554, - "acc_stderr": 0.026593939101844072, - "timestamp": "2024-11-18T13-06-35.949978" + "acc": 0.24074074074074073, + "acc_stderr": 0.026067159222275798, + "brier_score": 0.7641323203116751, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-54-06.552984" }, "arc_challenge": { "acc": 0.36945392491467577, diff --git a/metadata/model_scores/results_cerebras__Cerebras-GPT-1.3B.json b/metadata/model_scores/results_cerebras__Cerebras-GPT-1.3B.json index 23abdc1..33a77cc 100644 --- a/metadata/model_scores/results_cerebras__Cerebras-GPT-1.3B.json +++ b/metadata/model_scores/results_cerebras__Cerebras-GPT-1.3B.json @@ -1,6 +1,6 @@ { "model_name": "cerebras/Cerebras-GPT-1.3B", - "last_updated": "2024-12-04 11:24:08.682755", + "last_updated": "2024-12-19 13:39:39.590011", "results": { "harness": { "minerva_math_precalc": { @@ -228,289 +228,403 @@ "timestamp": "2024-06-11T18-40-02.959579" }, "mmlu_world_religions": { - "acc": 0.30409356725146197, - "acc_stderr": 0.03528211258245231, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.32748538011695905, + "acc_stderr": 0.03599335771456027, + "brier_score": 0.8937719714888455, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_formal_logic": { - "acc": 0.2619047619047619, - "acc_stderr": 0.039325376803928724, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.2698412698412698, + "acc_stderr": 0.03970158273235172, + "brier_score": 0.8046229494807222, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_prehistory": { - "acc": 0.2623456790123457, - "acc_stderr": 0.024477222856135124, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.21296296296296297, + "acc_stderr": 0.022779719088733396, + "brier_score": 0.9238815756889868, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_moral_scenarios": { - "acc": 0.23687150837988827, - "acc_stderr": 0.014219570788103982, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.24134078212290502, + "acc_stderr": 0.014310999547961455, + "brier_score": 0.7654012206749515, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_high_school_world_history": { "acc": 0.2742616033755274, - "acc_stderr": 0.029041333510598046, - "timestamp": "2024-11-21T01-52-17.626746" + "acc_stderr": 0.029041333510598042, + "brier_score": 0.8953044137931278, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_moral_disputes": { - "acc": 0.26878612716763006, - "acc_stderr": 0.02386800326250011, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.24855491329479767, + "acc_stderr": 0.023267528432100174, + "brier_score": 0.9076762471398718, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_professional_law": { - "acc": 0.2379400260756193, - "acc_stderr": 0.01087570078769423, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.24445893089960888, + "acc_stderr": 0.010976425013113909, + "brier_score": 0.9081932458376115, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_logical_fallacies": { - "acc": 0.3006134969325153, - "acc_stderr": 0.03602511318806771, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.22699386503067484, + "acc_stderr": 0.03291099578615768, + "brier_score": 0.9047494006062847, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_high_school_us_history": { - "acc": 0.22549019607843138, - "acc_stderr": 0.029331162294251728, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.25, + "acc_stderr": 0.03039153369274154, + "brier_score": 0.8896682536103456, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_philosophy": { - "acc": 0.26688102893890675, - "acc_stderr": 0.025122637608816643, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.1864951768488746, + "acc_stderr": 0.02212243977248077, + "brier_score": 0.9950397047163916, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_jurisprudence": { - "acc": 0.24074074074074073, - "acc_stderr": 0.041331194402438376, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.25, + "acc_stderr": 0.04186091791394607, + "brier_score": 0.9152850991695487, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_international_law": { - "acc": 0.2727272727272727, - "acc_stderr": 0.04065578140908705, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.2396694214876033, + "acc_stderr": 0.03896878985070417, + "brier_score": 1.0013452528417237, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_high_school_european_history": { - "acc": 0.26666666666666666, - "acc_stderr": 0.03453131801885415, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.21212121212121213, + "acc_stderr": 0.03192271569548301, + "brier_score": 0.8834985641364104, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_high_school_government_and_politics": { - "acc": 0.34196891191709844, - "acc_stderr": 0.03423465100104282, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.20207253886010362, + "acc_stderr": 0.02897908979429673, + "brier_score": 0.97165754636985, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_high_school_microeconomics": { - "acc": 0.22268907563025211, - "acc_stderr": 0.027025433498882357, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.21008403361344538, + "acc_stderr": 0.026461398717471874, + "brier_score": 0.934970235522194, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_high_school_geography": { - "acc": 0.31313131313131315, - "acc_stderr": 0.03304205087813652, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.17676767676767677, + "acc_stderr": 0.027178752639044915, + "brier_score": 0.9866251603319629, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_high_school_psychology": { - "acc": 0.3431192660550459, - "acc_stderr": 0.020354777736086037, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.1908256880733945, + "acc_stderr": 0.01684767640009109, + "brier_score": 0.9465549801953812, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_public_relations": { - "acc": 0.16363636363636364, - "acc_stderr": 0.035434330542986794, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.20909090909090908, + "acc_stderr": 0.03895091015724137, + "brier_score": 0.9756479393225346, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_us_foreign_policy": { - "acc": 0.2, - "acc_stderr": 0.04020151261036843, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.27, + "acc_stderr": 0.0446196043338474, + "brier_score": 0.8913658520374631, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_sociology": { - "acc": 0.22885572139303484, - "acc_stderr": 0.02970528405677245, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.24378109452736318, + "acc_stderr": 0.030360490154014652, + "brier_score": 0.9723441679448906, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_high_school_macroeconomics": { - "acc": 0.3230769230769231, - "acc_stderr": 0.023710888501970565, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.2, + "acc_stderr": 0.020280805062535726, + "brier_score": 0.9225384105640989, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_security_studies": { - "acc": 0.23265306122448978, - "acc_stderr": 0.02704925791589618, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.18775510204081633, + "acc_stderr": 0.02500025603954622, + "brier_score": 0.9870533074312139, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_professional_psychology": { - "acc": 0.272875816993464, - "acc_stderr": 0.01802047414839358, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.25163398692810457, + "acc_stderr": 0.01755581809132225, + "brier_score": 0.8988898696397957, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_human_sexuality": { - "acc": 0.2366412213740458, - "acc_stderr": 0.03727673575596918, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.2595419847328244, + "acc_stderr": 0.03844876139785271, + "brier_score": 0.8984801627491069, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_econometrics": { - "acc": 0.24561403508771928, - "acc_stderr": 0.04049339297748141, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.23684210526315788, + "acc_stderr": 0.039994238792813386, + "brier_score": 0.9009249067332376, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_miscellaneous": { - "acc": 0.24776500638569604, - "acc_stderr": 0.015438083080568965, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.23754789272030652, + "acc_stderr": 0.015218733046150195, + "brier_score": 0.9171301184162767, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_marketing": { "acc": 0.2905982905982906, "acc_stderr": 0.029745048572674057, - "timestamp": "2024-11-21T01-52-17.626746" + "brier_score": 0.8904287868983547, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_management": { - "acc": 0.20388349514563106, - "acc_stderr": 0.0398913985953177, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.18446601941747573, + "acc_stderr": 0.03840423627288276, + "brier_score": 1.0585623359693057, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_nutrition": { - "acc": 0.2973856209150327, - "acc_stderr": 0.02617390850671858, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.22875816993464052, + "acc_stderr": 0.024051029739912258, + "brier_score": 0.9635607731453667, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_medical_genetics": { - "acc": 0.28, - "acc_stderr": 0.04512608598542127, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "brier_score": 0.8623705842444132, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_human_aging": { - "acc": 0.242152466367713, - "acc_stderr": 0.028751392398694755, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.31390134529147984, + "acc_stderr": 0.03114679648297246, + "brier_score": 0.8158083435782157, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_professional_medicine": { - "acc": 0.41911764705882354, - "acc_stderr": 0.02997280717046463, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.1875, + "acc_stderr": 0.023709788253811766, + "brier_score": 1.0112612383958777, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_college_medicine": { - "acc": 0.2774566473988439, - "acc_stderr": 0.034140140070440354, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.20809248554913296, + "acc_stderr": 0.030952890217749884, + "brier_score": 0.9331850115461513, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_business_ethics": { - "acc": 0.22, - "acc_stderr": 0.041633319989322716, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "brier_score": 0.853814068235312, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_clinical_knowledge": { - "acc": 0.27169811320754716, - "acc_stderr": 0.027377706624670713, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.21509433962264152, + "acc_stderr": 0.025288394502891377, + "brier_score": 1.0002391981534213, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_global_facts": { - "acc": 0.31, - "acc_stderr": 0.04648231987117316, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.18, + "acc_stderr": 0.038612291966536955, + "brier_score": 0.9287312274934277, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_virology": { - "acc": 0.3313253012048193, - "acc_stderr": 0.03664314777288085, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.28313253012048195, + "acc_stderr": 0.03507295431370518, + "brier_score": 0.8980442362410103, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_professional_accounting": { - "acc": 0.2765957446808511, - "acc_stderr": 0.026684564340461008, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.22695035460992907, + "acc_stderr": 0.024987106365642973, + "brier_score": 0.886959008863655, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_college_physics": { - "acc": 0.22549019607843138, - "acc_stderr": 0.04158307533083286, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.21568627450980393, + "acc_stderr": 0.040925639582376556, + "brier_score": 0.8534210516503284, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_high_school_physics": { - "acc": 0.23841059602649006, - "acc_stderr": 0.0347918557259966, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.2052980132450331, + "acc_stderr": 0.032979866484738336, + "brier_score": 0.8619739918680707, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_high_school_biology": { - "acc": 0.2032258064516129, - "acc_stderr": 0.022891687984554963, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.1774193548387097, + "acc_stderr": 0.021732540689329265, + "brier_score": 0.9335354805317831, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_college_biology": { - "acc": 0.2361111111111111, - "acc_stderr": 0.03551446610810826, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.25, + "acc_stderr": 0.03621034121889507, + "brier_score": 0.891799674376725, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_anatomy": { - "acc": 0.2074074074074074, - "acc_stderr": 0.03502553170678317, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.18518518518518517, + "acc_stderr": 0.03355677216313142, + "brier_score": 1.0232390398107596, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_college_chemistry": { - "acc": 0.24, - "acc_stderr": 0.04292346959909284, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.19, + "acc_stderr": 0.03942772444036623, + "brier_score": 0.8849749719428637, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_computer_security": { - "acc": 0.21, - "acc_stderr": 0.04093601807403325, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.28, + "acc_stderr": 0.045126085985421276, + "brier_score": 0.8346834446161736, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_college_computer_science": { - "acc": 0.34, - "acc_stderr": 0.04760952285695235, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.22, + "acc_stderr": 0.041633319989322695, + "brier_score": 0.8582590301713728, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_astronomy": { - "acc": 0.21052631578947367, - "acc_stderr": 0.033176727875331574, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.17105263157894737, + "acc_stderr": 0.030643607071677098, + "brier_score": 0.8904045198086842, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_college_mathematics": { - "acc": 0.31, - "acc_stderr": 0.04648231987117316, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.22, + "acc_stderr": 0.041633319989322695, + "brier_score": 0.8630935255597413, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_conceptual_physics": { - "acc": 0.2851063829787234, - "acc_stderr": 0.02951319662553935, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.2680851063829787, + "acc_stderr": 0.028957342788342347, + "brier_score": 0.8508397492714637, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_abstract_algebra": { - "acc": 0.25, - "acc_stderr": 0.04351941398892446, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.22, + "acc_stderr": 0.04163331998932269, + "brier_score": 0.8737331790827914, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_high_school_computer_science": { - "acc": 0.29, - "acc_stderr": 0.04560480215720684, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 0.8761422947323227, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_machine_learning": { - "acc": 0.29464285714285715, - "acc_stderr": 0.04327040932578728, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.32142857142857145, + "acc_stderr": 0.04432804055291519, + "brier_score": 0.8139880556182322, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_high_school_chemistry": { - "acc": 0.23645320197044334, - "acc_stderr": 0.029896114291733552, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.15270935960591134, + "acc_stderr": 0.025308904539380627, + "brier_score": 0.9375182997733705, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_high_school_statistics": { - "acc": 0.4722222222222222, - "acc_stderr": 0.0340470532865388, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.16203703703703703, + "acc_stderr": 0.02513045365226846, + "brier_score": 0.923847541062819, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_elementary_mathematics": { - "acc": 0.24074074074074073, - "acc_stderr": 0.0220190800122179, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.20899470899470898, + "acc_stderr": 0.020940481565334835, + "brier_score": 0.8580255403499754, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_electrical_engineering": { - "acc": 0.23448275862068965, - "acc_stderr": 0.035306258743465914, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.2413793103448276, + "acc_stderr": 0.03565998174135302, + "brier_score": 0.866950272038678, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "mmlu_high_school_mathematics": { - "acc": 0.26296296296296295, - "acc_stderr": 0.026842057873833706, - "timestamp": "2024-11-21T01-52-17.626746" + "acc": 0.21851851851851853, + "acc_stderr": 0.02519575225182379, + "brier_score": 0.8407321696951865, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-52-10.280607" }, "arc_challenge": { "acc": 0.24744027303754265, diff --git a/metadata/model_scores/results_cerebras__Cerebras-GPT-2.7B.json b/metadata/model_scores/results_cerebras__Cerebras-GPT-2.7B.json index 285c7da..1f1129a 100644 --- a/metadata/model_scores/results_cerebras__Cerebras-GPT-2.7B.json +++ b/metadata/model_scores/results_cerebras__Cerebras-GPT-2.7B.json @@ -1,6 +1,6 @@ { "model_name": "cerebras/Cerebras-GPT-2.7B", - "last_updated": "2024-12-04 11:24:35.537170", + "last_updated": "2024-12-19 13:40:29.112136", "results": { "harness": { "minerva_math_precalc": { @@ -228,289 +228,403 @@ "timestamp": "2024-06-11T17-43-37.534801" }, "mmlu_world_religions": { - "acc": 0.26900584795321636, - "acc_stderr": 0.0340105262010409, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.3333333333333333, + "acc_stderr": 0.03615507630310935, + "brier_score": 0.7476930573228682, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_formal_logic": { - "acc": 0.2222222222222222, - "acc_stderr": 0.03718489006818114, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.18253968253968253, + "acc_stderr": 0.03455071019102147, + "brier_score": 0.7756165109851708, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_prehistory": { - "acc": 0.23765432098765432, - "acc_stderr": 0.02368359183700856, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.22839506172839505, + "acc_stderr": 0.023358211840626267, + "brier_score": 0.7923180096902048, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_moral_scenarios": { - "acc": 0.27150837988826815, - "acc_stderr": 0.014874252168095271, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.23798882681564246, + "acc_stderr": 0.014242630070574885, + "brier_score": 0.7560601311261977, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_high_school_world_history": { "acc": 0.25738396624472576, - "acc_stderr": 0.028458820991460288, - "timestamp": "2024-11-07T10-27-52.672195" + "acc_stderr": 0.028458820991460295, + "brier_score": 0.7574709963177562, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_moral_disputes": { - "acc": 0.26878612716763006, - "acc_stderr": 0.023868003262500104, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.2543352601156069, + "acc_stderr": 0.02344582627654555, + "brier_score": 0.7863398783390683, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_professional_law": { - "acc": 0.2633637548891786, - "acc_stderr": 0.011249506403605284, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.258148631029987, + "acc_stderr": 0.011176923719313397, + "brier_score": 0.7834504398219595, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_logical_fallacies": { - "acc": 0.31901840490797545, - "acc_stderr": 0.03661997551073836, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.2147239263803681, + "acc_stderr": 0.03226219377286774, + "brier_score": 0.7749737316033809, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_high_school_us_history": { - "acc": 0.2696078431372549, - "acc_stderr": 0.031145570659486782, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.20588235294117646, + "acc_stderr": 0.028379449451588667, + "brier_score": 0.7730385015882367, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_philosophy": { - "acc": 0.27009646302250806, - "acc_stderr": 0.025218040373410622, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.19935691318327975, + "acc_stderr": 0.022691033780549656, + "brier_score": 0.8200560029886024, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_jurisprudence": { - "acc": 0.18518518518518517, - "acc_stderr": 0.037552658650371835, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.28703703703703703, + "acc_stderr": 0.04373313040914761, + "brier_score": 0.781378343322981, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_international_law": { - "acc": 0.2396694214876033, - "acc_stderr": 0.03896878985070417, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.24793388429752067, + "acc_stderr": 0.039418975265163025, + "brier_score": 0.7768501435131513, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_high_school_european_history": { - "acc": 0.22424242424242424, - "acc_stderr": 0.03256866661681102, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.21818181818181817, + "acc_stderr": 0.03225078108306289, + "brier_score": 0.7776240256500807, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_high_school_government_and_politics": { "acc": 0.21761658031088082, "acc_stderr": 0.02977866303775296, - "timestamp": "2024-11-07T10-27-52.672195" + "brier_score": 0.7999517058536764, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_high_school_microeconomics": { - "acc": 0.23529411764705882, - "acc_stderr": 0.027553614467863804, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.20588235294117646, + "acc_stderr": 0.026265024608275882, + "brier_score": 0.7860553522313176, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_high_school_geography": { - "acc": 0.22727272727272727, - "acc_stderr": 0.02985751567338641, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.19696969696969696, + "acc_stderr": 0.02833560973246335, + "brier_score": 0.8000606853746892, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_high_school_psychology": { - "acc": 0.23302752293577983, - "acc_stderr": 0.018125669180861503, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.23853211009174313, + "acc_stderr": 0.01827257581023187, + "brier_score": 0.7696250170238662, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_public_relations": { - "acc": 0.16363636363636364, - "acc_stderr": 0.035434330542986774, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.20909090909090908, + "acc_stderr": 0.03895091015724137, + "brier_score": 0.7735527423605206, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_us_foreign_policy": { - "acc": 0.29, - "acc_stderr": 0.04560480215720683, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.28, + "acc_stderr": 0.04512608598542128, + "brier_score": 0.7691975852419901, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_sociology": { - "acc": 0.2537313432835821, - "acc_stderr": 0.03076944496729601, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.22388059701492538, + "acc_stderr": 0.029475250236017193, + "brier_score": 0.8032467232239617, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_high_school_macroeconomics": { - "acc": 0.24102564102564103, - "acc_stderr": 0.021685546665333195, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.2128205128205128, + "acc_stderr": 0.020752423722128027, + "brier_score": 0.7881431774671177, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_security_studies": { - "acc": 0.32653061224489793, - "acc_stderr": 0.030021056238440307, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.2, + "acc_stderr": 0.025607375986579153, + "brier_score": 0.7926343764752054, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_professional_psychology": { - "acc": 0.2581699346405229, - "acc_stderr": 0.017704531653250078, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.25980392156862747, + "acc_stderr": 0.017740899509177795, + "brier_score": 0.7721471100019798, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_human_sexuality": { - "acc": 0.2366412213740458, - "acc_stderr": 0.037276735755969174, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.24427480916030533, + "acc_stderr": 0.03768335959728744, + "brier_score": 0.7740153632877359, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_econometrics": { - "acc": 0.24561403508771928, - "acc_stderr": 0.040493392977481425, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.23684210526315788, + "acc_stderr": 0.039994238792813344, + "brier_score": 0.7599040869797429, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_miscellaneous": { - "acc": 0.26181353767560667, - "acc_stderr": 0.01572083867844527, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.280970625798212, + "acc_stderr": 0.016073127851221235, + "brier_score": 0.7657858959409427, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_marketing": { - "acc": 0.2564102564102564, - "acc_stderr": 0.02860595370200425, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.31196581196581197, + "acc_stderr": 0.03035152732334493, + "brier_score": 0.7525991848995963, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_management": { - "acc": 0.22330097087378642, - "acc_stderr": 0.04123553189891431, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.2912621359223301, + "acc_stderr": 0.044986763205729224, + "brier_score": 0.7733049014327111, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_nutrition": { - "acc": 0.2679738562091503, - "acc_stderr": 0.025360603796242553, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.22549019607843138, + "acc_stderr": 0.023929155517351284, + "brier_score": 0.7840834700358891, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_medical_genetics": { - "acc": 0.21, - "acc_stderr": 0.040936018074033256, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.29, + "acc_stderr": 0.04560480215720683, + "brier_score": 0.7580529964562934, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_human_aging": { - "acc": 0.242152466367713, - "acc_stderr": 0.028751392398694755, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.29596412556053814, + "acc_stderr": 0.0306365913486998, + "brier_score": 0.7562483844482, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_professional_medicine": { - "acc": 0.25, - "acc_stderr": 0.026303648393696036, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.17279411764705882, + "acc_stderr": 0.02296606758558178, + "brier_score": 0.776164881685414, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_college_medicine": { - "acc": 0.23121387283236994, - "acc_stderr": 0.032147373020294696, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.23699421965317918, + "acc_stderr": 0.03242414757483099, + "brier_score": 0.7794146792011398, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_business_ethics": { - "acc": 0.22, - "acc_stderr": 0.04163331998932269, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "brier_score": 0.7558271806493747, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_clinical_knowledge": { - "acc": 0.21509433962264152, - "acc_stderr": 0.025288394502891373, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.20754716981132076, + "acc_stderr": 0.02495991802891127, + "brier_score": 0.7972495925387602, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_global_facts": { - "acc": 0.18, - "acc_stderr": 0.038612291966536955, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.2, + "acc_stderr": 0.04020151261036844, + "brier_score": 0.7775920033083306, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_virology": { - "acc": 0.28313253012048195, - "acc_stderr": 0.03507295431370519, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.2469879518072289, + "acc_stderr": 0.03357351982064536, + "brier_score": 0.7758619736761486, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_professional_accounting": { - "acc": 0.2695035460992908, - "acc_stderr": 0.026469036818590634, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.24468085106382978, + "acc_stderr": 0.025645553622266733, + "brier_score": 0.7720524096989436, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_college_physics": { - "acc": 0.17647058823529413, - "acc_stderr": 0.037932811853078105, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.2549019607843137, + "acc_stderr": 0.04336432707993179, + "brier_score": 0.749662073099727, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_high_school_physics": { - "acc": 0.2781456953642384, - "acc_stderr": 0.03658603262763743, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.2980132450331126, + "acc_stderr": 0.037345356767871984, + "brier_score": 0.7472706507270679, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_high_school_biology": { - "acc": 0.24838709677419354, - "acc_stderr": 0.024580028921481003, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.23548387096774193, + "acc_stderr": 0.02413763242933771, + "brier_score": 0.7690663879971471, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_college_biology": { "acc": 0.2708333333333333, - "acc_stderr": 0.03716177437566016, - "timestamp": "2024-11-07T10-27-52.672195" + "acc_stderr": 0.037161774375660164, + "brier_score": 0.7683342643108664, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_anatomy": { - "acc": 0.2814814814814815, - "acc_stderr": 0.03885004245800255, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.2222222222222222, + "acc_stderr": 0.035914440841969694, + "brier_score": 0.7780246383757022, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_college_chemistry": { - "acc": 0.19, - "acc_stderr": 0.03942772444036623, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.14, + "acc_stderr": 0.03487350880197771, + "brier_score": 0.7790738942811067, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_computer_security": { "acc": 0.26, - "acc_stderr": 0.0440844002276808, - "timestamp": "2024-11-07T10-27-52.672195" + "acc_stderr": 0.04408440022768079, + "brier_score": 0.7572718501114037, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_college_computer_science": { - "acc": 0.37, - "acc_stderr": 0.048523658709391, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.28, + "acc_stderr": 0.045126085985421296, + "brier_score": 0.7470019487908016, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_astronomy": { - "acc": 0.19736842105263158, - "acc_stderr": 0.03238981601699397, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.27631578947368424, + "acc_stderr": 0.03639057569952924, + "brier_score": 0.7481578957001758, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_college_mathematics": { - "acc": 0.25, - "acc_stderr": 0.04351941398892446, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "brier_score": 0.7615635626380802, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_conceptual_physics": { - "acc": 0.25957446808510637, - "acc_stderr": 0.028659179374292316, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.28085106382978725, + "acc_stderr": 0.029379170464124825, + "brier_score": 0.7747178620837013, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_abstract_algebra": { - "acc": 0.23, - "acc_stderr": 0.042295258468165044, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.22, + "acc_stderr": 0.041633319989322695, + "brier_score": 0.7949552265673998, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_high_school_computer_science": { - "acc": 0.39, - "acc_stderr": 0.04902071300001975, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.33, + "acc_stderr": 0.04725815626252604, + "brier_score": 0.736408727816062, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_machine_learning": { - "acc": 0.3392857142857143, - "acc_stderr": 0.04493949068613539, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.3482142857142857, + "acc_stderr": 0.04521829902833585, + "brier_score": 0.7449644713984921, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_high_school_chemistry": { - "acc": 0.2660098522167488, - "acc_stderr": 0.03108982600293752, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.24630541871921183, + "acc_stderr": 0.030315099285617736, + "brier_score": 0.7678783424089564, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_high_school_statistics": { - "acc": 0.2824074074074074, - "acc_stderr": 0.030701372111510927, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.19444444444444445, + "acc_stderr": 0.02699145450203673, + "brier_score": 0.7702784574532471, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_elementary_mathematics": { - "acc": 0.2037037037037037, - "acc_stderr": 0.020742740560122656, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.25925925925925924, + "acc_stderr": 0.02256989707491842, + "brier_score": 0.7596261911717037, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_electrical_engineering": { - "acc": 0.2482758620689655, - "acc_stderr": 0.036001056927277716, - "timestamp": "2024-11-07T10-27-52.672195" + "acc": 0.2413793103448276, + "acc_stderr": 0.03565998174135302, + "brier_score": 0.7637082530045226, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "mmlu_high_school_mathematics": { "acc": 0.26666666666666666, - "acc_stderr": 0.02696242432507384, - "timestamp": "2024-11-07T10-27-52.672195" + "acc_stderr": 0.02696242432507385, + "brier_score": 0.7582533211051161, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-09-24.727491" }, "arc_challenge": { "acc": 0.26109215017064846, diff --git a/metadata/model_scores/results_cerebras__Cerebras-GPT-6.7B.json b/metadata/model_scores/results_cerebras__Cerebras-GPT-6.7B.json index ff1fee1..01efa6c 100644 --- a/metadata/model_scores/results_cerebras__Cerebras-GPT-6.7B.json +++ b/metadata/model_scores/results_cerebras__Cerebras-GPT-6.7B.json @@ -1,6 +1,6 @@ { "model_name": "cerebras/Cerebras-GPT-6.7B", - "last_updated": "2024-12-04 11:25:50.673900", + "last_updated": "2024-12-19 13:42:07.428570", "results": { "harness": { "minerva_math_precalc": { @@ -228,289 +228,403 @@ "timestamp": "2024-06-11T16-09-16.235478" }, "mmlu_world_religions": { - "acc": 0.32748538011695905, - "acc_stderr": 0.03599335771456027, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.2046783625730994, + "acc_stderr": 0.03094445977853321, + "brier_score": 0.7957314835738306, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_formal_logic": { - "acc": 0.23809523809523808, - "acc_stderr": 0.03809523809523811, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.25396825396825395, + "acc_stderr": 0.03893259610604674, + "brier_score": 0.777821546810293, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_prehistory": { - "acc": 0.24074074074074073, - "acc_stderr": 0.02378858355165853, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.26851851851851855, + "acc_stderr": 0.02465968518596728, + "brier_score": 0.7605214100168148, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_moral_scenarios": { - "acc": 0.27262569832402234, - "acc_stderr": 0.014893391735249608, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.27039106145251396, + "acc_stderr": 0.014854993938010085, + "brier_score": 0.7743030187987928, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_high_school_world_history": { - "acc": 0.22362869198312235, - "acc_stderr": 0.027123298205229972, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.2489451476793249, + "acc_stderr": 0.028146970599422644, + "brier_score": 0.7759565012926257, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_moral_disputes": { - "acc": 0.28034682080924855, - "acc_stderr": 0.02418242749657762, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.2543352601156069, + "acc_stderr": 0.02344582627654555, + "brier_score": 0.7612048282171314, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_professional_law": { - "acc": 0.26988265971316816, - "acc_stderr": 0.01133738108425041, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.2620599739243807, + "acc_stderr": 0.011231552795890394, + "brier_score": 0.7637414528130592, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_logical_fallacies": { "acc": 0.27607361963190186, - "acc_stderr": 0.03512385283705051, - "timestamp": "2024-11-21T21-50-03.007594" + "acc_stderr": 0.0351238528370505, + "brier_score": 0.7592525580081232, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_high_school_us_history": { - "acc": 0.23529411764705882, - "acc_stderr": 0.029771775228145638, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.23039215686274508, + "acc_stderr": 0.029554292605695066, + "brier_score": 0.7726198780482679, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_philosophy": { - "acc": 0.2604501607717042, - "acc_stderr": 0.02492672322484554, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.21864951768488747, + "acc_stderr": 0.023475581417861106, + "brier_score": 0.7673678637362098, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_jurisprudence": { - "acc": 0.23148148148148148, - "acc_stderr": 0.04077494709252626, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.3055555555555556, + "acc_stderr": 0.044531975073749834, + "brier_score": 0.7565128770198866, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_international_law": { - "acc": 0.2644628099173554, - "acc_stderr": 0.04026187527591207, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.2727272727272727, + "acc_stderr": 0.04065578140908705, + "brier_score": 0.7638130866066336, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_high_school_european_history": { - "acc": 0.2727272727272727, - "acc_stderr": 0.03477691162163659, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.24242424242424243, + "acc_stderr": 0.03346409881055953, + "brier_score": 0.7687477534580277, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_high_school_government_and_politics": { - "acc": 0.21761658031088082, - "acc_stderr": 0.02977866303775296, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.21243523316062177, + "acc_stderr": 0.02951928261681725, + "brier_score": 0.7649253497346248, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_high_school_microeconomics": { - "acc": 0.24789915966386555, - "acc_stderr": 0.028047967224176892, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.23949579831932774, + "acc_stderr": 0.027722065493361276, + "brier_score": 0.7648537684862107, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_high_school_geography": { - "acc": 0.18686868686868688, - "acc_stderr": 0.027772533334218977, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.25252525252525254, + "acc_stderr": 0.030954055470365907, + "brier_score": 0.7515992458403655, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_high_school_psychology": { - "acc": 0.29908256880733947, - "acc_stderr": 0.01963041728541517, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.24770642201834864, + "acc_stderr": 0.018508143602547815, + "brier_score": 0.7572896033283785, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_public_relations": { - "acc": 0.24545454545454545, - "acc_stderr": 0.04122066502878285, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.34545454545454546, + "acc_stderr": 0.04554619617541054, + "brier_score": 0.7489090104905242, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_us_foreign_policy": { - "acc": 0.27, - "acc_stderr": 0.04461960433384739, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.28, + "acc_stderr": 0.04512608598542127, + "brier_score": 0.7485337935026642, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_sociology": { - "acc": 0.24875621890547264, - "acc_stderr": 0.030567675938916714, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.22885572139303484, + "acc_stderr": 0.029705284056772443, + "brier_score": 0.7896160133465063, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_high_school_macroeconomics": { - "acc": 0.2564102564102564, - "acc_stderr": 0.02213908110397153, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.2358974358974359, + "acc_stderr": 0.021525965407408726, + "brier_score": 0.7592758243699075, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_security_studies": { - "acc": 0.19591836734693877, - "acc_stderr": 0.025409301953225678, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.22040816326530613, + "acc_stderr": 0.02653704531214531, + "brier_score": 0.7651608116817085, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_professional_psychology": { - "acc": 0.24673202614379086, - "acc_stderr": 0.017440820367402493, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.24836601307189543, + "acc_stderr": 0.017479487001364764, + "brier_score": 0.7626463494252116, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_human_sexuality": { "acc": 0.25190839694656486, - "acc_stderr": 0.038073871163060866, - "timestamp": "2024-11-21T21-50-03.007594" + "acc_stderr": 0.03807387116306086, + "brier_score": 0.7803830593272657, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_econometrics": { - "acc": 0.24561403508771928, - "acc_stderr": 0.04049339297748142, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.2719298245614035, + "acc_stderr": 0.041857744240220575, + "brier_score": 0.7678727754466477, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_miscellaneous": { - "acc": 0.26436781609195403, - "acc_stderr": 0.01576998484069052, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.2771392081736909, + "acc_stderr": 0.016005636294122418, + "brier_score": 0.7670373181589404, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_marketing": { - "acc": 0.23931623931623933, - "acc_stderr": 0.027951826808924333, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.2606837606837607, + "acc_stderr": 0.028760348956523414, + "brier_score": 0.779345195412505, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_management": { - "acc": 0.27184466019417475, - "acc_stderr": 0.044052680241409216, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.24271844660194175, + "acc_stderr": 0.04245022486384495, + "brier_score": 0.7577522389013621, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_nutrition": { - "acc": 0.23529411764705882, - "acc_stderr": 0.02428861946604612, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.24183006535947713, + "acc_stderr": 0.024518195641879334, + "brier_score": 0.7696166584432055, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_medical_genetics": { - "acc": 0.26, - "acc_stderr": 0.04408440022768078, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.33, + "acc_stderr": 0.04725815626252605, + "brier_score": 0.7349891487761616, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_human_aging": { - "acc": 0.29596412556053814, - "acc_stderr": 0.030636591348699817, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.2914798206278027, + "acc_stderr": 0.030500283176545896, + "brier_score": 0.7480796284249333, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_professional_medicine": { - "acc": 0.41911764705882354, - "acc_stderr": 0.029972807170464622, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.4411764705882353, + "acc_stderr": 0.030161911930767102, + "brier_score": 0.7181797736284031, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_college_medicine": { - "acc": 0.24277456647398843, - "acc_stderr": 0.0326926380614177, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.2774566473988439, + "acc_stderr": 0.03414014007044036, + "brier_score": 0.7563416289746194, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_business_ethics": { - "acc": 0.14, - "acc_stderr": 0.03487350880197773, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 0.7785417779353457, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_clinical_knowledge": { - "acc": 0.23018867924528302, - "acc_stderr": 0.025907897122408173, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.2792452830188679, + "acc_stderr": 0.027611163402399715, + "brier_score": 0.7548416769314967, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_global_facts": { - "acc": 0.18, - "acc_stderr": 0.03861229196653697, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "brier_score": 0.7468815557293845, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_virology": { - "acc": 0.26506024096385544, - "acc_stderr": 0.03436024037944967, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.29518072289156627, + "acc_stderr": 0.0355092018568963, + "brier_score": 0.7496600717109347, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_professional_accounting": { - "acc": 0.2553191489361702, - "acc_stderr": 0.026011992930902013, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.25177304964539005, + "acc_stderr": 0.025892151156709405, + "brier_score": 0.7673164867592129, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_college_physics": { - "acc": 0.24509803921568626, - "acc_stderr": 0.042801058373643966, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.28431372549019607, + "acc_stderr": 0.04488482852329017, + "brier_score": 0.7516552649663509, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_high_school_physics": { - "acc": 0.271523178807947, - "acc_stderr": 0.03631329803969653, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.2582781456953642, + "acc_stderr": 0.035737053147634576, + "brier_score": 0.7498692234299653, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_high_school_biology": { - "acc": 0.21935483870967742, - "acc_stderr": 0.023540799358723295, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.267741935483871, + "acc_stderr": 0.025189006660212385, + "brier_score": 0.7474022289330349, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_college_biology": { - "acc": 0.20833333333333334, - "acc_stderr": 0.033961162058453336, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.2638888888888889, + "acc_stderr": 0.03685651095897532, + "brier_score": 0.7548767358146407, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_anatomy": { - "acc": 0.22962962962962963, - "acc_stderr": 0.036333844140734636, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.24444444444444444, + "acc_stderr": 0.037125378336148665, + "brier_score": 0.7607710979577496, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_college_chemistry": { - "acc": 0.31, - "acc_stderr": 0.04648231987117316, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "brier_score": 0.7695601415588362, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_computer_security": { - "acc": 0.21, - "acc_stderr": 0.040936018074033256, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.22, + "acc_stderr": 0.04163331998932269, + "brier_score": 0.780396000142865, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_college_computer_science": { - "acc": 0.4, - "acc_stderr": 0.049236596391733084, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "brier_score": 0.7623333845129798, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_astronomy": { - "acc": 0.2565789473684211, - "acc_stderr": 0.0355418036802569, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.23026315789473684, + "acc_stderr": 0.03426059424403165, + "brier_score": 0.773281846138913, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_college_mathematics": { - "acc": 0.3, - "acc_stderr": 0.046056618647183814, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.24, + "acc_stderr": 0.04292346959909283, + "brier_score": 0.7632750997930634, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_conceptual_physics": { - "acc": 0.3191489361702128, - "acc_stderr": 0.030472973363380056, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.28085106382978725, + "acc_stderr": 0.029379170464124815, + "brier_score": 0.7470199702981599, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_abstract_algebra": { - "acc": 0.28, - "acc_stderr": 0.04512608598542128, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "brier_score": 0.7659346829135254, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_high_school_computer_science": { - "acc": 0.37, - "acc_stderr": 0.048523658709391, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.23, + "acc_stderr": 0.04229525846816505, + "brier_score": 0.7693477530508929, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_machine_learning": { - "acc": 0.23214285714285715, - "acc_stderr": 0.04007341809755807, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.20535714285714285, + "acc_stderr": 0.03834241021419073, + "brier_score": 0.7733834786097267, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_high_school_chemistry": { - "acc": 0.20689655172413793, - "acc_stderr": 0.02850137816789395, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.29064039408866993, + "acc_stderr": 0.0319474007226554, + "brier_score": 0.7443894806007804, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_high_school_statistics": { - "acc": 0.3425925925925926, - "acc_stderr": 0.032365852526021574, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.25462962962962965, + "acc_stderr": 0.029711275860005357, + "brier_score": 0.7529905203543267, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_elementary_mathematics": { - "acc": 0.23015873015873015, - "acc_stderr": 0.021679219663693145, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.24603174603174602, + "acc_stderr": 0.022182037202948368, + "brier_score": 0.7755612883830363, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_electrical_engineering": { - "acc": 0.25517241379310346, - "acc_stderr": 0.03632984052707842, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.3103448275862069, + "acc_stderr": 0.038552896163789485, + "brier_score": 0.7485238433258922, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "mmlu_high_school_mathematics": { - "acc": 0.26296296296296295, - "acc_stderr": 0.026842057873833713, - "timestamp": "2024-11-21T21-50-03.007594" + "acc": 0.2814814814814815, + "acc_stderr": 0.027420019350945273, + "brier_score": 0.7568005107125074, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-35-15.687187" }, "arc_challenge": { "acc": 0.3242320819112628, diff --git a/metadata/model_scores/results_cerebras__btlm-3b-8k-base.json b/metadata/model_scores/results_cerebras__btlm-3b-8k-base.json index 3b8fc21..fa6c73d 100644 --- a/metadata/model_scores/results_cerebras__btlm-3b-8k-base.json +++ b/metadata/model_scores/results_cerebras__btlm-3b-8k-base.json @@ -1,6 +1,6 @@ { "model_name": "cerebras/btlm-3b-8k-base", - "last_updated": "2024-12-04 11:22:30.192230", + "last_updated": "2024-12-19 13:37:47.471855", "results": { "harness": { "minerva_math_precalc": { @@ -228,289 +228,403 @@ "timestamp": "2024-06-13T13-48-42.209852" }, "mmlu_world_religions": { - "acc": 0.30409356725146197, - "acc_stderr": 0.03528211258245231, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.29239766081871343, + "acc_stderr": 0.034886477134579215, + "brier_score": 0.765283458975345, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_formal_logic": { - "acc": 0.21428571428571427, - "acc_stderr": 0.03670066451047181, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.19047619047619047, + "acc_stderr": 0.03512207412302054, + "brier_score": 0.7811181241262286, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_prehistory": { - "acc": 0.2808641975308642, - "acc_stderr": 0.025006469755799208, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.29012345679012347, + "acc_stderr": 0.025251173936495026, + "brier_score": 0.7525978131388321, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_moral_scenarios": { - "acc": 0.24581005586592178, - "acc_stderr": 0.014400296429225619, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.24692737430167597, + "acc_stderr": 0.014422292204808838, + "brier_score": 0.806961001008417, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_high_school_world_history": { - "acc": 0.33755274261603374, - "acc_stderr": 0.030781549102026223, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.270042194092827, + "acc_stderr": 0.028900721906293426, + "brier_score": 0.7661484483752257, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_moral_disputes": { - "acc": 0.2861271676300578, - "acc_stderr": 0.024332146779134128, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.2976878612716763, + "acc_stderr": 0.024617055388677003, + "brier_score": 0.7513136133716709, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_professional_law": { - "acc": 0.2529335071707953, - "acc_stderr": 0.011102268713839987, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.27640156453715775, + "acc_stderr": 0.01142215319455358, + "brier_score": 0.7698465557943386, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_logical_fallacies": { - "acc": 0.2147239263803681, - "acc_stderr": 0.03226219377286774, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.3312883435582822, + "acc_stderr": 0.03697983910025588, + "brier_score": 0.7507690419290711, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_high_school_us_history": { "acc": 0.2549019607843137, - "acc_stderr": 0.030587591351604243, - "timestamp": "2024-11-22T14-26-18.572446" + "acc_stderr": 0.030587591351604246, + "brier_score": 0.7642473421486193, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_philosophy": { - "acc": 0.2765273311897106, - "acc_stderr": 0.025403832978179604, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.31189710610932475, + "acc_stderr": 0.02631185807185416, + "brier_score": 0.7518637246830614, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_jurisprudence": { "acc": 0.26851851851851855, "acc_stderr": 0.04284467968052191, - "timestamp": "2024-11-22T14-26-18.572446" + "brier_score": 0.7724755566562874, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_international_law": { - "acc": 0.2644628099173554, - "acc_stderr": 0.04026187527591205, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.371900826446281, + "acc_stderr": 0.044120158066245044, + "brier_score": 0.7278607764002772, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_high_school_european_history": { - "acc": 0.296969696969697, - "acc_stderr": 0.03567969772268047, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.2787878787878788, + "acc_stderr": 0.03501438706296782, + "brier_score": 0.7578314247618458, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_high_school_government_and_politics": { - "acc": 0.27461139896373055, - "acc_stderr": 0.03221024508041156, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.23834196891191708, + "acc_stderr": 0.03074890536390988, + "brier_score": 0.7556016591155795, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_high_school_microeconomics": { - "acc": 0.25630252100840334, - "acc_stderr": 0.02835962087053395, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.23949579831932774, + "acc_stderr": 0.02772206549336127, + "brier_score": 0.7543189745804204, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_high_school_geography": { - "acc": 0.22727272727272727, - "acc_stderr": 0.029857515673386414, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.25252525252525254, + "acc_stderr": 0.030954055470365907, + "brier_score": 0.7654693814686194, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_high_school_psychology": { - "acc": 0.26238532110091745, - "acc_stderr": 0.018861885021534745, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.24587155963302754, + "acc_stderr": 0.018461940968708443, + "brier_score": 0.760696893950433, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_public_relations": { - "acc": 0.2818181818181818, - "acc_stderr": 0.043091187099464585, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.19090909090909092, + "acc_stderr": 0.03764425585984927, + "brier_score": 0.7899351526404036, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_us_foreign_policy": { - "acc": 0.29, - "acc_stderr": 0.04560480215720684, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.28, + "acc_stderr": 0.045126085985421276, + "brier_score": 0.7540244111010149, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_sociology": { - "acc": 0.3034825870646766, - "acc_stderr": 0.03251006816458619, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.25870646766169153, + "acc_stderr": 0.03096590312357303, + "brier_score": 0.7590506093520147, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_high_school_macroeconomics": { - "acc": 0.24102564102564103, - "acc_stderr": 0.02168554666533319, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.24358974358974358, + "acc_stderr": 0.021763733684173933, + "brier_score": 0.7591500438875595, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_security_studies": { - "acc": 0.3510204081632653, - "acc_stderr": 0.03055531675557364, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.22857142857142856, + "acc_stderr": 0.026882144922307748, + "brier_score": 0.7515111354054198, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_professional_psychology": { - "acc": 0.27124183006535946, - "acc_stderr": 0.017986615304030316, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.28594771241830064, + "acc_stderr": 0.018280485072954687, + "brier_score": 0.7591990945166102, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_human_sexuality": { - "acc": 0.25190839694656486, - "acc_stderr": 0.038073871163060866, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.3053435114503817, + "acc_stderr": 0.04039314978724561, + "brier_score": 0.7547846920764133, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_econometrics": { - "acc": 0.2543859649122807, - "acc_stderr": 0.040969851398436716, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.21052631578947367, + "acc_stderr": 0.038351539543994194, + "brier_score": 0.7843401099086258, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_miscellaneous": { - "acc": 0.31928480204342274, - "acc_stderr": 0.01667126174953872, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.33588761174968074, + "acc_stderr": 0.016889407235171683, + "brier_score": 0.7393151258344807, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_marketing": { - "acc": 0.28205128205128205, - "acc_stderr": 0.02948036054954119, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.2692307692307692, + "acc_stderr": 0.029058588303748845, + "brier_score": 0.7582287865784976, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_management": { - "acc": 0.20388349514563106, - "acc_stderr": 0.0398913985953177, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.21359223300970873, + "acc_stderr": 0.04058042015646035, + "brier_score": 0.7677446200491856, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_nutrition": { - "acc": 0.25163398692810457, - "acc_stderr": 0.0248480182638752, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.27450980392156865, + "acc_stderr": 0.025553169991826517, + "brier_score": 0.759851624640113, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_medical_genetics": { - "acc": 0.3, - "acc_stderr": 0.046056618647183814, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "brier_score": 0.7706140139510501, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_human_aging": { - "acc": 0.3901345291479821, - "acc_stderr": 0.03273766725459157, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.22869955156950672, + "acc_stderr": 0.028188240046929196, + "brier_score": 0.7987091115125714, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_professional_medicine": { - "acc": 0.27205882352941174, - "acc_stderr": 0.027033041151681456, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.15808823529411764, + "acc_stderr": 0.02216146260806852, + "brier_score": 0.7779534874194103, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_college_medicine": { - "acc": 0.24277456647398843, - "acc_stderr": 0.0326926380614177, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.23699421965317918, + "acc_stderr": 0.03242414757483098, + "brier_score": 0.766554717412249, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_business_ethics": { - "acc": 0.31, - "acc_stderr": 0.04648231987117316, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.23, + "acc_stderr": 0.04229525846816505, + "brier_score": 0.7656375808021179, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_clinical_knowledge": { - "acc": 0.2339622641509434, - "acc_stderr": 0.02605529690115292, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.23773584905660378, + "acc_stderr": 0.02619980880756193, + "brier_score": 0.7631703136393346, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_global_facts": { - "acc": 0.32, - "acc_stderr": 0.04688261722621505, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.33, + "acc_stderr": 0.047258156262526045, + "brier_score": 0.7372744149477343, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_virology": { - "acc": 0.3132530120481928, - "acc_stderr": 0.036108050180310235, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.21084337349397592, + "acc_stderr": 0.03175554786629919, + "brier_score": 0.7607599505428622, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_professional_accounting": { - "acc": 0.24468085106382978, - "acc_stderr": 0.02564555362226673, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.2765957446808511, + "acc_stderr": 0.026684564340461, + "brier_score": 0.7716867889954537, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_college_physics": { - "acc": 0.2549019607843137, - "acc_stderr": 0.04336432707993177, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.21568627450980393, + "acc_stderr": 0.04092563958237656, + "brier_score": 0.7879728501816907, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_high_school_physics": { - "acc": 0.23178807947019867, - "acc_stderr": 0.034454062719870546, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.32450331125827814, + "acc_stderr": 0.038227469376587525, + "brier_score": 0.7381303399059673, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_high_school_biology": { - "acc": 0.23870967741935484, - "acc_stderr": 0.024251071262208837, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.267741935483871, + "acc_stderr": 0.025189006660212374, + "brier_score": 0.7432912408175562, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_college_biology": { - "acc": 0.3055555555555556, - "acc_stderr": 0.03852084696008534, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.2916666666666667, + "acc_stderr": 0.038009680605548574, + "brier_score": 0.75721550669838, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_anatomy": { - "acc": 0.2074074074074074, - "acc_stderr": 0.03502553170678318, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.34074074074074073, + "acc_stderr": 0.040943762699967946, + "brier_score": 0.7483348312450915, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_college_chemistry": { - "acc": 0.27, - "acc_stderr": 0.04461960433384741, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.24, + "acc_stderr": 0.04292346959909283, + "brier_score": 0.7596323870355468, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_computer_security": { - "acc": 0.26, - "acc_stderr": 0.04408440022768076, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "brier_score": 0.7524614960670815, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_college_computer_science": { - "acc": 0.3, - "acc_stderr": 0.046056618647183814, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.29, + "acc_stderr": 0.045604802157206845, + "brier_score": 0.7580127277762976, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_astronomy": { - "acc": 0.2631578947368421, - "acc_stderr": 0.035834961763610645, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.3223684210526316, + "acc_stderr": 0.03803510248351586, + "brier_score": 0.7482452724551685, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_college_mathematics": { - "acc": 0.3, - "acc_stderr": 0.046056618647183814, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.32, + "acc_stderr": 0.046882617226215034, + "brier_score": 0.7536100014431494, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_conceptual_physics": { - "acc": 0.2978723404255319, - "acc_stderr": 0.029896145682095462, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.2553191489361702, + "acc_stderr": 0.028504856470514203, + "brier_score": 0.7538476797353884, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_abstract_algebra": { - "acc": 0.26, - "acc_stderr": 0.04408440022768079, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "brier_score": 0.7643901098719358, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_high_school_computer_science": { - "acc": 0.23, - "acc_stderr": 0.042295258468165065, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.32, + "acc_stderr": 0.046882617226215034, + "brier_score": 0.7550698213030492, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_machine_learning": { - "acc": 0.25892857142857145, - "acc_stderr": 0.04157751539865629, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.24107142857142858, + "acc_stderr": 0.04059867246952687, + "brier_score": 0.7783521858574265, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_high_school_chemistry": { - "acc": 0.22660098522167488, - "acc_stderr": 0.029454863835292968, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.2955665024630542, + "acc_stderr": 0.032104944337514575, + "brier_score": 0.7512368230132239, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_high_school_statistics": { - "acc": 0.25462962962962965, - "acc_stderr": 0.029711275860005357, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.23148148148148148, + "acc_stderr": 0.028765111718046944, + "brier_score": 0.7474000718882282, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_elementary_mathematics": { "acc": 0.2619047619047619, - "acc_stderr": 0.022644212615525214, - "timestamp": "2024-11-22T14-26-18.572446" + "acc_stderr": 0.022644212615525218, + "brier_score": 0.7793545696195081, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_electrical_engineering": { - "acc": 0.21379310344827587, - "acc_stderr": 0.034165204477475494, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.27586206896551724, + "acc_stderr": 0.037245636197746325, + "brier_score": 0.7511138343126749, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "mmlu_high_school_mathematics": { - "acc": 0.25555555555555554, - "acc_stderr": 0.026593939101844065, - "timestamp": "2024-11-22T14-26-18.572446" + "acc": 0.25925925925925924, + "acc_stderr": 0.026719240783712173, + "brier_score": 0.7673250316212422, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-50-53.762839" }, "arc_challenge": { "acc": 0.38310580204778155, diff --git a/metadata/model_scores/results_facebook__opt-2.7b.json b/metadata/model_scores/results_facebook__opt-2.7b.json index 099ff50..c22ce90 100644 --- a/metadata/model_scores/results_facebook__opt-2.7b.json +++ b/metadata/model_scores/results_facebook__opt-2.7b.json @@ -1,6 +1,6 @@ { "model_name": "facebook/opt-2.7b", - "last_updated": "2024-12-04 11:25:45.756688", + "last_updated": "2024-12-19 13:42:00.789242", "results": { "harness": { "minerva_math_precalc": { @@ -228,289 +228,403 @@ "timestamp": "2024-06-06T12-08-58.689654" }, "mmlu_world_religions": { - "acc": 0.2222222222222222, - "acc_stderr": 0.03188578017686398, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.23391812865497075, + "acc_stderr": 0.03246721765117828, + "brier_score": 0.7878935837815728, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_formal_logic": { - "acc": 0.19047619047619047, - "acc_stderr": 0.035122074123020534, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.21428571428571427, + "acc_stderr": 0.03670066451047182, + "brier_score": 0.7677482439006217, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_prehistory": { - "acc": 0.25308641975308643, - "acc_stderr": 0.024191808600713002, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.2962962962962963, + "acc_stderr": 0.025407197798890155, + "brier_score": 0.7564602866799459, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_moral_scenarios": { - "acc": 0.2346368715083799, - "acc_stderr": 0.014173044098303673, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.25139664804469275, + "acc_stderr": 0.01450897945355397, + "brier_score": 0.7658966744392373, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_high_school_world_history": { - "acc": 0.24472573839662448, - "acc_stderr": 0.027985699387036423, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.25738396624472576, + "acc_stderr": 0.028458820991460288, + "brier_score": 0.7797932422503456, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_moral_disputes": { - "acc": 0.2543352601156069, - "acc_stderr": 0.02344582627654554, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.2861271676300578, + "acc_stderr": 0.02433214677913413, + "brier_score": 0.7667677037603893, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_professional_law": { - "acc": 0.2503259452411995, - "acc_stderr": 0.01106415102716544, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.2516297262059974, + "acc_stderr": 0.011083276280441905, + "brier_score": 0.7707280847501268, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_logical_fallacies": { - "acc": 0.2331288343558282, - "acc_stderr": 0.0332201579577674, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.3496932515337423, + "acc_stderr": 0.03746668325470023, + "brier_score": 0.7615028914354622, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_high_school_us_history": { - "acc": 0.2549019607843137, - "acc_stderr": 0.030587591351604246, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.2696078431372549, + "acc_stderr": 0.031145570659486782, + "brier_score": 0.766031691820315, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_philosophy": { - "acc": 0.3247588424437299, - "acc_stderr": 0.026596782287697046, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.3054662379421222, + "acc_stderr": 0.026160584450140485, + "brier_score": 0.7496992228361217, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_jurisprudence": { - "acc": 0.25, - "acc_stderr": 0.04186091791394607, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.24074074074074073, + "acc_stderr": 0.04133119440243839, + "brier_score": 0.7733523691001012, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_international_law": { - "acc": 0.24793388429752067, - "acc_stderr": 0.039418975265163025, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.34710743801652894, + "acc_stderr": 0.04345724570292534, + "brier_score": 0.7423423252676877, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_high_school_european_history": { - "acc": 0.23030303030303031, - "acc_stderr": 0.03287666758603488, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.2787878787878788, + "acc_stderr": 0.03501438706296781, + "brier_score": 0.7671193652171602, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_high_school_government_and_politics": { - "acc": 0.35233160621761656, - "acc_stderr": 0.03447478286414357, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.3005181347150259, + "acc_stderr": 0.0330881859441575, + "brier_score": 0.754098993058133, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_high_school_microeconomics": { - "acc": 0.2184873949579832, - "acc_stderr": 0.02684151432295895, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.24789915966386555, + "acc_stderr": 0.028047967224176896, + "brier_score": 0.7569618511834054, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_high_school_geography": { - "acc": 0.21212121212121213, - "acc_stderr": 0.029126522834586815, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.2474747474747475, + "acc_stderr": 0.03074630074212452, + "brier_score": 0.7602981396993943, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_high_school_psychology": { - "acc": 0.3155963302752294, - "acc_stderr": 0.01992611751386967, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.22752293577981653, + "acc_stderr": 0.017974463578776502, + "brier_score": 0.7592180999483095, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_public_relations": { - "acc": 0.22727272727272727, - "acc_stderr": 0.04013964554072773, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.24545454545454545, + "acc_stderr": 0.041220665028782855, + "brier_score": 0.7570539380574031, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_us_foreign_policy": { - "acc": 0.25, - "acc_stderr": 0.04351941398892446, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.22, + "acc_stderr": 0.04163331998932269, + "brier_score": 0.7705241951464472, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_sociology": { - "acc": 0.263681592039801, - "acc_stderr": 0.031157150869355568, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.25870646766169153, + "acc_stderr": 0.030965903123573026, + "brier_score": 0.7533474420589423, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_high_school_macroeconomics": { - "acc": 0.358974358974359, - "acc_stderr": 0.024321738484602357, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.22564102564102564, + "acc_stderr": 0.02119363252514854, + "brier_score": 0.7593425818521468, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_security_studies": { - "acc": 0.2163265306122449, - "acc_stderr": 0.026358916334904024, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.2653061224489796, + "acc_stderr": 0.028263889943784603, + "brier_score": 0.7580450235688722, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_professional_psychology": { - "acc": 0.2679738562091503, - "acc_stderr": 0.017917974069594722, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.25980392156862747, + "acc_stderr": 0.0177408995091778, + "brier_score": 0.7616363188926287, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_human_sexuality": { - "acc": 0.2366412213740458, - "acc_stderr": 0.03727673575596918, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.24427480916030533, + "acc_stderr": 0.03768335959728744, + "brier_score": 0.7711793914236916, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_econometrics": { - "acc": 0.24561403508771928, - "acc_stderr": 0.04049339297748141, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.21052631578947367, + "acc_stderr": 0.03835153954399421, + "brier_score": 0.767892062589693, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_miscellaneous": { - "acc": 0.23371647509578544, - "acc_stderr": 0.015133383278988829, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.24904214559386972, + "acc_stderr": 0.015464676163395951, + "brier_score": 0.7727186122170707, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_marketing": { - "acc": 0.23076923076923078, - "acc_stderr": 0.027601921381417618, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.2692307692307692, + "acc_stderr": 0.029058588303748842, + "brier_score": 0.7682489849238419, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_management": { - "acc": 0.3883495145631068, - "acc_stderr": 0.0482572933735639, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.2524271844660194, + "acc_stderr": 0.04301250399690878, + "brier_score": 0.7675660008154402, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_nutrition": { - "acc": 0.23202614379084968, - "acc_stderr": 0.02417084087934101, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.25163398692810457, + "acc_stderr": 0.024848018263875195, + "brier_score": 0.7631766135670024, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_medical_genetics": { - "acc": 0.33, - "acc_stderr": 0.04725815626252604, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.23, + "acc_stderr": 0.042295258468165065, + "brier_score": 0.7673701717101398, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_human_aging": { - "acc": 0.20179372197309417, - "acc_stderr": 0.026936111912802273, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.19730941704035873, + "acc_stderr": 0.02670985334496796, + "brier_score": 0.8047248026269651, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_professional_medicine": { - "acc": 0.33455882352941174, - "acc_stderr": 0.02866199620233531, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.16544117647058823, + "acc_stderr": 0.022571771025494757, + "brier_score": 0.7947207604837336, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_college_medicine": { - "acc": 0.23699421965317918, - "acc_stderr": 0.03242414757483098, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.24855491329479767, + "acc_stderr": 0.03295304696818318, + "brier_score": 0.7645409283951206, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_business_ethics": { - "acc": 0.19, - "acc_stderr": 0.03942772444036624, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.27, + "acc_stderr": 0.044619604333847394, + "brier_score": 0.767232076633074, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_clinical_knowledge": { - "acc": 0.21132075471698114, - "acc_stderr": 0.025125766484827845, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.23018867924528302, + "acc_stderr": 0.02590789712240817, + "brier_score": 0.7653957956215449, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_global_facts": { - "acc": 0.36, - "acc_stderr": 0.04824181513244218, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.29, + "acc_stderr": 0.04560480215720684, + "brier_score": 0.7629692025072817, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_virology": { - "acc": 0.21686746987951808, - "acc_stderr": 0.032082844503563655, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.22289156626506024, + "acc_stderr": 0.03240004825594686, + "brier_score": 0.7743406704718713, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_professional_accounting": { - "acc": 0.2872340425531915, - "acc_stderr": 0.026992199173064356, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.26595744680851063, + "acc_stderr": 0.02635806569888059, + "brier_score": 0.7646076852898195, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_college_physics": { "acc": 0.21568627450980393, "acc_stderr": 0.04092563958237656, - "timestamp": "2024-11-21T14-28-39.469398" + "brier_score": 0.7529136910088495, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_high_school_physics": { - "acc": 0.31125827814569534, - "acc_stderr": 0.03780445850526733, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.26490066225165565, + "acc_stderr": 0.036030385453603826, + "brier_score": 0.739006262173674, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_high_school_biology": { - "acc": 0.2161290322580645, - "acc_stderr": 0.023415293433568532, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.25483870967741934, + "acc_stderr": 0.024790118459332208, + "brier_score": 0.7589112207960926, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_college_biology": { - "acc": 0.2152777777777778, - "acc_stderr": 0.03437079344106135, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.2013888888888889, + "acc_stderr": 0.03353647469713841, + "brier_score": 0.7666553641033335, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_anatomy": { - "acc": 0.2814814814814815, - "acc_stderr": 0.03885004245800254, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.32592592592592595, + "acc_stderr": 0.040491220417025055, + "brier_score": 0.7317195380722945, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_college_chemistry": { - "acc": 0.21, - "acc_stderr": 0.040936018074033256, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.23, + "acc_stderr": 0.042295258468165065, + "brier_score": 0.7765942042217988, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_computer_security": { - "acc": 0.26, - "acc_stderr": 0.0440844002276808, - "timestamp": "2024-11-21T14-28-39.469398" - }, - "mmlu_college_computer_science": { "acc": 0.3, "acc_stderr": 0.046056618647183814, - "timestamp": "2024-11-21T14-28-39.469398" + "brier_score": 0.7561788166964886, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" + }, + "mmlu_college_computer_science": { + "acc": 0.34, + "acc_stderr": 0.04760952285695236, + "brier_score": 0.7529124410986495, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_astronomy": { - "acc": 0.19078947368421054, - "acc_stderr": 0.031975658210325004, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.23684210526315788, + "acc_stderr": 0.03459777606810537, + "brier_score": 0.7606069163529254, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_college_mathematics": { - "acc": 0.23, - "acc_stderr": 0.04229525846816505, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.28, + "acc_stderr": 0.04512608598542129, + "brier_score": 0.7567010659399342, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_conceptual_physics": { - "acc": 0.23829787234042554, - "acc_stderr": 0.0278512529738898, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.225531914893617, + "acc_stderr": 0.02732107841738753, + "brier_score": 0.7863535098918605, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_abstract_algebra": { - "acc": 0.21, - "acc_stderr": 0.040936018074033256, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.26, + "acc_stderr": 0.04408440022768079, + "brier_score": 0.7497993014925278, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_high_school_computer_science": { - "acc": 0.34, - "acc_stderr": 0.04760952285695236, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.33, + "acc_stderr": 0.04725815626252605, + "brier_score": 0.7427906928398942, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_machine_learning": { - "acc": 0.25892857142857145, - "acc_stderr": 0.04157751539865629, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.25, + "acc_stderr": 0.04109974682633932, + "brier_score": 0.7688521974534093, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_high_school_chemistry": { - "acc": 0.270935960591133, - "acc_stderr": 0.031270907132976984, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.3251231527093596, + "acc_stderr": 0.03295797566311271, + "brier_score": 0.7415130211082905, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_high_school_statistics": { - "acc": 0.39814814814814814, - "acc_stderr": 0.03338473403207402, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.2638888888888889, + "acc_stderr": 0.03005820270430985, + "brier_score": 0.747827956637948, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_elementary_mathematics": { - "acc": 0.24074074074074073, - "acc_stderr": 0.022019080012217883, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.23809523809523808, + "acc_stderr": 0.021935878081184756, + "brier_score": 0.7646395040474879, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_electrical_engineering": { "acc": 0.2827586206896552, "acc_stderr": 0.03752833958003336, - "timestamp": "2024-11-21T14-28-39.469398" + "brier_score": 0.7630012012413374, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "mmlu_high_school_mathematics": { - "acc": 0.2851851851851852, - "acc_stderr": 0.027528599210340496, - "timestamp": "2024-11-21T14-28-39.469398" + "acc": 0.3037037037037037, + "acc_stderr": 0.02803792996911499, + "brier_score": 0.7425845374832011, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-48-44.848780" }, "arc_challenge": { "acc": 0.31399317406143346, diff --git a/metadata/model_scores/results_facebook__opt-350m.json b/metadata/model_scores/results_facebook__opt-350m.json index 337b568..fd62aeb 100644 --- a/metadata/model_scores/results_facebook__opt-350m.json +++ b/metadata/model_scores/results_facebook__opt-350m.json @@ -1,6 +1,6 @@ { "model_name": "facebook/opt-350m", - "last_updated": "2024-12-04 11:25:31.530440", + "last_updated": "2024-12-19 13:41:43.373740", "results": { "harness": { "minerva_math_precalc": { @@ -228,289 +228,403 @@ "timestamp": "2024-06-06T08-49-49.969484" }, "mmlu_world_religions": { - "acc": 0.18128654970760233, - "acc_stderr": 0.02954774168764002, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.3157894736842105, + "acc_stderr": 0.03565079670708313, + "brier_score": 0.9970922337481368, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_formal_logic": { - "acc": 0.23015873015873015, - "acc_stderr": 0.03764950879790605, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.2777777777777778, + "acc_stderr": 0.04006168083848876, + "brier_score": 0.9990126853115562, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_prehistory": { - "acc": 0.2716049382716049, - "acc_stderr": 0.024748624490537382, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.2222222222222222, + "acc_stderr": 0.02313237623454333, + "brier_score": 0.938124319036566, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_moral_scenarios": { - "acc": 0.24022346368715083, - "acc_stderr": 0.014288343803925307, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.2670391061452514, + "acc_stderr": 0.014796502622562557, + "brier_score": 0.8040521732226942, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_high_school_world_history": { - "acc": 0.20675105485232068, - "acc_stderr": 0.0263616516683891, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.2911392405063291, + "acc_stderr": 0.029571601065753374, + "brier_score": 0.8014140166309869, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_moral_disputes": { - "acc": 0.26878612716763006, - "acc_stderr": 0.023868003262500107, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.23699421965317918, + "acc_stderr": 0.02289408248992599, + "brier_score": 0.8942909106290047, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_professional_law": { - "acc": 0.23989569752281617, - "acc_stderr": 0.010906282617981634, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.2392438070404172, + "acc_stderr": 0.010896123652676644, + "brier_score": 0.8333329190374003, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_logical_fallacies": { - "acc": 0.24539877300613497, - "acc_stderr": 0.03380939813943354, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.2085889570552147, + "acc_stderr": 0.03192193448934722, + "brier_score": 0.936184462271342, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_high_school_us_history": { - "acc": 0.25980392156862747, - "acc_stderr": 0.03077855467869328, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.23529411764705882, + "acc_stderr": 0.02977177522814563, + "brier_score": 0.8922406630766441, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_philosophy": { - "acc": 0.18971061093247588, - "acc_stderr": 0.022268196258783218, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.20257234726688103, + "acc_stderr": 0.022827317491059675, + "brier_score": 0.9655163469797318, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_jurisprudence": { - "acc": 0.2222222222222222, - "acc_stderr": 0.040191074725573483, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.25925925925925924, + "acc_stderr": 0.042365112580946336, + "brier_score": 0.9051404598397155, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_international_law": { - "acc": 0.371900826446281, - "acc_stderr": 0.04412015806624504, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.23140495867768596, + "acc_stderr": 0.03849856098794088, + "brier_score": 0.9717857105693258, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_high_school_european_history": { - "acc": 0.26666666666666666, - "acc_stderr": 0.03453131801885415, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.23030303030303031, + "acc_stderr": 0.0328766675860349, + "brier_score": 0.8574131059083935, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_high_school_government_and_politics": { - "acc": 0.3160621761658031, - "acc_stderr": 0.03355397369686173, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.16580310880829016, + "acc_stderr": 0.026839845022314415, + "brier_score": 0.9750618207520899, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_high_school_microeconomics": { - "acc": 0.2773109243697479, - "acc_stderr": 0.029079374539480007, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.226890756302521, + "acc_stderr": 0.027205371538279472, + "brier_score": 0.9519753931469569, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_high_school_geography": { - "acc": 0.35858585858585856, - "acc_stderr": 0.03416903640391521, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.18686868686868688, + "acc_stderr": 0.027772533334218964, + "brier_score": 1.0463696895724388, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_high_school_psychology": { - "acc": 0.3431192660550459, - "acc_stderr": 0.02035477773608604, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.1981651376146789, + "acc_stderr": 0.017090573804217902, + "brier_score": 0.9547817682010158, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_public_relations": { - "acc": 0.20909090909090908, - "acc_stderr": 0.038950910157241364, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.22727272727272727, + "acc_stderr": 0.040139645540727735, + "brier_score": 0.935406570724006, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_us_foreign_policy": { - "acc": 0.26, - "acc_stderr": 0.0440844002276808, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.27, + "acc_stderr": 0.0446196043338474, + "brier_score": 0.9232569027624487, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_sociology": { - "acc": 0.2537313432835821, - "acc_stderr": 0.030769444967296004, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.22388059701492538, + "acc_stderr": 0.029475250236017193, + "brier_score": 0.9329688170951506, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_high_school_macroeconomics": { - "acc": 0.2794871794871795, - "acc_stderr": 0.022752388839776826, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.20512820512820512, + "acc_stderr": 0.02047323317355198, + "brier_score": 0.9727437139626431, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_security_studies": { - "acc": 0.4, - "acc_stderr": 0.031362502409358936, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.18775510204081633, + "acc_stderr": 0.025000256039546212, + "brier_score": 0.9654425189494698, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_professional_psychology": { - "acc": 0.23529411764705882, - "acc_stderr": 0.017160587235046345, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.2581699346405229, + "acc_stderr": 0.017704531653250075, + "brier_score": 0.8587422968758852, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_human_sexuality": { "acc": 0.26717557251908397, - "acc_stderr": 0.038808483010823944, - "timestamp": "2024-11-22T15-23-45.050539" + "acc_stderr": 0.03880848301082396, + "brier_score": 0.8835713271957959, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_econometrics": { "acc": 0.22807017543859648, "acc_stderr": 0.03947152782669415, - "timestamp": "2024-11-22T15-23-45.050539" + "brier_score": 1.0031390195254952, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_miscellaneous": { - "acc": 0.20945083014048532, - "acc_stderr": 0.0145513105681437, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.23371647509578544, + "acc_stderr": 0.015133383278988846, + "brier_score": 1.0278711513007095, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_marketing": { - "acc": 0.20512820512820512, - "acc_stderr": 0.026453508054040332, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.28205128205128205, + "acc_stderr": 0.02948036054954119, + "brier_score": 0.8246799820943679, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_management": { "acc": 0.1941747572815534, - "acc_stderr": 0.03916667762822584, - "timestamp": "2024-11-22T15-23-45.050539" + "acc_stderr": 0.03916667762822585, + "brier_score": 1.046568863939919, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_nutrition": { - "acc": 0.2581699346405229, - "acc_stderr": 0.025058503316958147, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.20915032679738563, + "acc_stderr": 0.023287685312334813, + "brier_score": 0.9709268398131645, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_medical_genetics": { - "acc": 0.31, - "acc_stderr": 0.04648231987117316, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.35, + "acc_stderr": 0.047937248544110196, + "brier_score": 0.8803520332018766, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_human_aging": { - "acc": 0.16143497757847533, - "acc_stderr": 0.024693957899128472, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.3183856502242152, + "acc_stderr": 0.03126580522513713, + "brier_score": 0.7888401526589168, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_professional_medicine": { - "acc": 0.4485294117647059, - "acc_stderr": 0.030211479609121593, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.1801470588235294, + "acc_stderr": 0.02334516361654487, + "brier_score": 0.9400302557350523, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_college_medicine": { - "acc": 0.23699421965317918, - "acc_stderr": 0.03242414757483098, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.19653179190751446, + "acc_stderr": 0.03029957466478814, + "brier_score": 0.9596453548624214, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_business_ethics": { - "acc": 0.21, - "acc_stderr": 0.040936018074033256, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.27, + "acc_stderr": 0.044619604333847394, + "brier_score": 0.8593432203954157, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_clinical_knowledge": { - "acc": 0.26037735849056604, - "acc_stderr": 0.02700876609070808, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.2037735849056604, + "acc_stderr": 0.024790784501775416, + "brier_score": 0.9704544573365829, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_global_facts": { - "acc": 0.19, - "acc_stderr": 0.039427724440366234, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.18, + "acc_stderr": 0.038612291966536955, + "brier_score": 1.20802224351284, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_virology": { - "acc": 0.18674698795180722, - "acc_stderr": 0.030338749144500594, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.30120481927710846, + "acc_stderr": 0.0357160923005348, + "brier_score": 0.8459372985636292, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_professional_accounting": { - "acc": 0.24468085106382978, - "acc_stderr": 0.025645553622266733, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.2375886524822695, + "acc_stderr": 0.025389512552729903, + "brier_score": 0.9314436226712202, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_college_physics": { - "acc": 0.22549019607843138, - "acc_stderr": 0.04158307533083286, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.21568627450980393, + "acc_stderr": 0.040925639582376556, + "brier_score": 1.0911694886311318, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_high_school_physics": { - "acc": 0.33774834437086093, - "acc_stderr": 0.03861557546255168, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.1986754966887417, + "acc_stderr": 0.032578473844367746, + "brier_score": 1.0475729127219355, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_high_school_biology": { - "acc": 0.2967741935483871, - "acc_stderr": 0.02598850079241189, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.18387096774193548, + "acc_stderr": 0.02203721734026784, + "brier_score": 0.9483958221465675, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_college_biology": { - "acc": 0.20833333333333334, - "acc_stderr": 0.033961162058453336, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.25, + "acc_stderr": 0.03621034121889507, + "brier_score": 0.8893980700028072, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_anatomy": { - "acc": 0.26666666666666666, - "acc_stderr": 0.038201699145179055, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.2074074074074074, + "acc_stderr": 0.03502553170678317, + "brier_score": 0.91315436761325, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_college_chemistry": { - "acc": 0.35, - "acc_stderr": 0.0479372485441102, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.19, + "acc_stderr": 0.03942772444036623, + "brier_score": 1.1052586190340865, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_computer_security": { - "acc": 0.18, - "acc_stderr": 0.03861229196653697, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.33, + "acc_stderr": 0.047258156262526045, + "brier_score": 0.8464448149776044, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_college_computer_science": { - "acc": 0.29, - "acc_stderr": 0.045604802157206845, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 1.0344961178099206, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_astronomy": { "acc": 0.17763157894736842, "acc_stderr": 0.031103182383123398, - "timestamp": "2024-11-22T15-23-45.050539" + "brier_score": 1.047975313862721, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_college_mathematics": { - "acc": 0.27, - "acc_stderr": 0.044619604333847394, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "brier_score": 1.163913051137866, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_conceptual_physics": { - "acc": 0.30638297872340425, - "acc_stderr": 0.030135906478517563, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.2851063829787234, + "acc_stderr": 0.029513196625539355, + "brier_score": 0.8870185367021838, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_abstract_algebra": { - "acc": 0.22, - "acc_stderr": 0.04163331998932269, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "brier_score": 1.1577966267866362, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_high_school_computer_science": { - "acc": 0.19, - "acc_stderr": 0.03942772444036625, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 0.9710461070416393, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_machine_learning": { - "acc": 0.23214285714285715, - "acc_stderr": 0.04007341809755805, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.3125, + "acc_stderr": 0.043994650575715215, + "brier_score": 0.8944828792710238, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_high_school_chemistry": { - "acc": 0.2955665024630542, - "acc_stderr": 0.032104944337514575, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.15763546798029557, + "acc_stderr": 0.0256390141311724, + "brier_score": 1.0681987273212483, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_high_school_statistics": { - "acc": 0.4722222222222222, - "acc_stderr": 0.0340470532865388, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.16203703703703703, + "acc_stderr": 0.02513045365226846, + "brier_score": 1.103831661183318, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_elementary_mathematics": { - "acc": 0.2566137566137566, - "acc_stderr": 0.022494510767503154, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.20899470899470898, + "acc_stderr": 0.020940481565334835, + "brier_score": 1.2024783942465607, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_electrical_engineering": { - "acc": 0.31724137931034485, - "acc_stderr": 0.038783523721386236, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.25517241379310346, + "acc_stderr": 0.03632984052707842, + "brier_score": 0.927153451586835, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "mmlu_high_school_mathematics": { - "acc": 0.27037037037037037, - "acc_stderr": 0.027080372815145658, - "timestamp": "2024-11-22T15-23-45.050539" + "acc": 0.2111111111111111, + "acc_stderr": 0.02488211685765508, + "brier_score": 1.1781786016679823, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-45-02.639673" }, "arc_challenge": { "acc": 0.20733788395904437, diff --git a/metadata/model_scores/results_facebook__opt-6.7b.json b/metadata/model_scores/results_facebook__opt-6.7b.json index 138327d..6dad661 100644 --- a/metadata/model_scores/results_facebook__opt-6.7b.json +++ b/metadata/model_scores/results_facebook__opt-6.7b.json @@ -1,6 +1,6 @@ { "model_name": "facebook/opt-6.7b", - "last_updated": "2024-12-04 11:25:01.792655", + "last_updated": "2024-12-19 13:41:04.593268", "results": { "harness": { "minerva_math_precalc": { @@ -109,289 +109,403 @@ "timestamp": "2024-11-21T16-41-59.547447" }, "mmlu_world_religions": { - "acc": 0.2222222222222222, - "acc_stderr": 0.03188578017686398, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.2573099415204678, + "acc_stderr": 0.03352799844161865, + "brier_score": 0.7552209591883933, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_formal_logic": { - "acc": 0.16666666666666666, - "acc_stderr": 0.03333333333333337, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.1984126984126984, + "acc_stderr": 0.035670166752768614, + "brier_score": 0.7737676613415517, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_prehistory": { - "acc": 0.27469135802469136, - "acc_stderr": 0.024836057868294674, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.2808641975308642, + "acc_stderr": 0.025006469755799208, + "brier_score": 0.7550137099246275, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_moral_scenarios": { - "acc": 0.2424581005586592, - "acc_stderr": 0.014333522059217887, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.24692737430167597, + "acc_stderr": 0.01442229220480884, + "brier_score": 0.7847262404434647, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_high_school_world_history": { "acc": 0.25738396624472576, "acc_stderr": 0.0284588209914603, - "timestamp": "2024-11-21T16-41-59.547447" + "brier_score": 0.7608715004399518, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_moral_disputes": { - "acc": 0.23699421965317918, - "acc_stderr": 0.02289408248992599, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.26878612716763006, + "acc_stderr": 0.023868003262500097, + "brier_score": 0.7530167216408132, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_professional_law": { - "acc": 0.23533246414602346, - "acc_stderr": 0.010834432543912238, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.2685788787483703, + "acc_stderr": 0.011320056629121734, + "brier_score": 0.7745934802801677, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_logical_fallacies": { - "acc": 0.24539877300613497, - "acc_stderr": 0.03380939813943354, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.2883435582822086, + "acc_stderr": 0.03559039531617342, + "brier_score": 0.7562371230137225, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_high_school_us_history": { - "acc": 0.24019607843137256, - "acc_stderr": 0.02998373305591361, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.22549019607843138, + "acc_stderr": 0.029331162294251735, + "brier_score": 0.7606373179788259, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_philosophy": { - "acc": 0.2282958199356913, - "acc_stderr": 0.023839303311398215, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.2090032154340836, + "acc_stderr": 0.023093140398374224, + "brier_score": 0.7619163388633432, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_jurisprudence": { - "acc": 0.25, - "acc_stderr": 0.04186091791394607, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.21296296296296297, + "acc_stderr": 0.03957835471980981, + "brier_score": 0.7544655022994979, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_international_law": { - "acc": 0.256198347107438, - "acc_stderr": 0.03984979653302872, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.371900826446281, + "acc_stderr": 0.04412015806624504, + "brier_score": 0.738750265298197, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_high_school_european_history": { - "acc": 0.24242424242424243, - "acc_stderr": 0.033464098810559534, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.21818181818181817, + "acc_stderr": 0.03225078108306289, + "brier_score": 0.7596660922455263, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_high_school_government_and_politics": { "acc": 0.23316062176165803, "acc_stderr": 0.03051611137147601, - "timestamp": "2024-11-21T16-41-59.547447" + "brier_score": 0.7755860661040752, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_high_school_microeconomics": { - "acc": 0.1722689075630252, - "acc_stderr": 0.024528664971305417, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.20588235294117646, + "acc_stderr": 0.026265024608275882, + "brier_score": 0.7775274996254388, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_high_school_geography": { - "acc": 0.19696969696969696, - "acc_stderr": 0.02833560973246335, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.20707070707070707, + "acc_stderr": 0.02886977846026705, + "brier_score": 0.770352204182849, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_high_school_psychology": { - "acc": 0.26972477064220185, - "acc_stderr": 0.019028486711115452, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.23486238532110093, + "acc_stderr": 0.018175110510343588, + "brier_score": 0.7657261725880065, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_public_relations": { - "acc": 0.32727272727272727, - "acc_stderr": 0.0449429086625209, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.3181818181818182, + "acc_stderr": 0.044612721759105085, + "brier_score": 0.7584306917793805, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_us_foreign_policy": { - "acc": 0.22, - "acc_stderr": 0.041633319989322695, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.23, + "acc_stderr": 0.042295258468165065, + "brier_score": 0.7602175169099713, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_sociology": { - "acc": 0.23383084577114427, - "acc_stderr": 0.029929415408348377, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.22388059701492538, + "acc_stderr": 0.029475250236017197, + "brier_score": 0.7672048853971253, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_high_school_macroeconomics": { - "acc": 0.24358974358974358, - "acc_stderr": 0.021763733684173926, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.20256410256410257, + "acc_stderr": 0.020377660970371383, + "brier_score": 0.7789187510682279, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_security_studies": { - "acc": 0.19183673469387755, - "acc_stderr": 0.025206963154225392, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.23673469387755103, + "acc_stderr": 0.02721283588407316, + "brier_score": 0.7692639395936833, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_professional_psychology": { - "acc": 0.25163398692810457, - "acc_stderr": 0.017555818091322267, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.2696078431372549, + "acc_stderr": 0.017952449196987866, + "brier_score": 0.7547884905782574, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_human_sexuality": { - "acc": 0.22137404580152673, - "acc_stderr": 0.0364129708131373, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.24427480916030533, + "acc_stderr": 0.03768335959728744, + "brier_score": 0.7678727074555681, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_econometrics": { - "acc": 0.23684210526315788, - "acc_stderr": 0.039994238792813344, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.24561403508771928, + "acc_stderr": 0.04049339297748142, + "brier_score": 0.7614188863435601, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_miscellaneous": { - "acc": 0.2835249042145594, - "acc_stderr": 0.016117318166832276, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.26436781609195403, + "acc_stderr": 0.015769984840690518, + "brier_score": 0.7540259391362917, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_marketing": { - "acc": 0.2948717948717949, - "acc_stderr": 0.02987257770889117, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.28205128205128205, + "acc_stderr": 0.02948036054954119, + "brier_score": 0.7532337077509424, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_management": { "acc": 0.24271844660194175, - "acc_stderr": 0.04245022486384495, - "timestamp": "2024-11-21T16-41-59.547447" + "acc_stderr": 0.04245022486384493, + "brier_score": 0.7671254179408622, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_nutrition": { - "acc": 0.2222222222222222, - "acc_stderr": 0.023805186524888135, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.2647058823529412, + "acc_stderr": 0.025261691219729484, + "brier_score": 0.7612413425517579, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_medical_genetics": { - "acc": 0.32, - "acc_stderr": 0.046882617226215034, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.27, + "acc_stderr": 0.044619604333847394, + "brier_score": 0.7505691471150036, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_human_aging": { - "acc": 0.3542600896860987, - "acc_stderr": 0.032100621541349864, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.242152466367713, + "acc_stderr": 0.028751392398694755, + "brier_score": 0.7536864157975758, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_professional_medicine": { - "acc": 0.2647058823529412, - "acc_stderr": 0.02679956202488768, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.16176470588235295, + "acc_stderr": 0.022368672562886754, + "brier_score": 0.7799692276903291, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_college_medicine": { - "acc": 0.2543352601156069, - "acc_stderr": 0.0332055644308557, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.2947976878612717, + "acc_stderr": 0.034765996075164785, + "brier_score": 0.7619002765455879, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_business_ethics": { - "acc": 0.2, - "acc_stderr": 0.04020151261036845, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.24, + "acc_stderr": 0.042923469599092816, + "brier_score": 0.7504553219270095, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_clinical_knowledge": { - "acc": 0.18867924528301888, - "acc_stderr": 0.02407999513006224, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.21509433962264152, + "acc_stderr": 0.025288394502891366, + "brier_score": 0.7661011361188558, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_global_facts": { - "acc": 0.29, - "acc_stderr": 0.045604802157206845, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.27, + "acc_stderr": 0.044619604333847394, + "brier_score": 0.7482995931248143, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_virology": { - "acc": 0.3313253012048193, - "acc_stderr": 0.03664314777288086, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.26506024096385544, + "acc_stderr": 0.03436024037944967, + "brier_score": 0.7506023198628184, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_professional_accounting": { - "acc": 0.30141843971631205, - "acc_stderr": 0.027374128882631153, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.2730496453900709, + "acc_stderr": 0.02657786094330785, + "brier_score": 0.7542366976498601, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_college_physics": { "acc": 0.21568627450980393, - "acc_stderr": 0.04092563958237655, - "timestamp": "2024-11-21T16-41-59.547447" + "acc_stderr": 0.04092563958237656, + "brier_score": 0.7781365912609894, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_high_school_physics": { - "acc": 0.16556291390728478, - "acc_stderr": 0.03034818341030361, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.2582781456953642, + "acc_stderr": 0.035737053147634576, + "brier_score": 0.7707688373280365, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_high_school_biology": { - "acc": 0.23225806451612904, - "acc_stderr": 0.02402225613030824, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.2129032258064516, + "acc_stderr": 0.023287665127268518, + "brier_score": 0.7678464225240904, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_college_biology": { - "acc": 0.2222222222222222, - "acc_stderr": 0.03476590104304134, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.2638888888888889, + "acc_stderr": 0.03685651095897532, + "brier_score": 0.7660270777982109, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_anatomy": { - "acc": 0.3333333333333333, - "acc_stderr": 0.04072314811876837, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.3111111111111111, + "acc_stderr": 0.03999262876617723, + "brier_score": 0.7503755164314315, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_college_chemistry": { - "acc": 0.24, - "acc_stderr": 0.042923469599092816, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.2, + "acc_stderr": 0.04020151261036845, + "brier_score": 0.7845496059378259, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_computer_security": { - "acc": 0.24, - "acc_stderr": 0.04292346959909283, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "brier_score": 0.7566757002307685, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_college_computer_science": { - "acc": 0.3, - "acc_stderr": 0.046056618647183814, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.34, + "acc_stderr": 0.04760952285695235, + "brier_score": 0.7504260902356191, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_astronomy": { - "acc": 0.15789473684210525, - "acc_stderr": 0.029674167520101446, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.28289473684210525, + "acc_stderr": 0.03665349695640767, + "brier_score": 0.7661924313226414, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_college_mathematics": { - "acc": 0.3, - "acc_stderr": 0.046056618647183814, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.29, + "acc_stderr": 0.045604802157206845, + "brier_score": 0.7525634009133456, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_conceptual_physics": { - "acc": 0.2553191489361702, - "acc_stderr": 0.02850485647051418, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.22127659574468084, + "acc_stderr": 0.027136349602424066, + "brier_score": 0.7635802144135204, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_abstract_algebra": { "acc": 0.24, - "acc_stderr": 0.04292346959909283, - "timestamp": "2024-11-21T16-41-59.547447" + "acc_stderr": 0.04292346959909282, + "brier_score": 0.7477432389158422, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_high_school_computer_science": { - "acc": 0.24, - "acc_stderr": 0.042923469599092816, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.34, + "acc_stderr": 0.04760952285695236, + "brier_score": 0.7467316005783833, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_machine_learning": { - "acc": 0.29464285714285715, - "acc_stderr": 0.04327040932578728, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.25892857142857145, + "acc_stderr": 0.04157751539865629, + "brier_score": 0.7397021806186821, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_high_school_chemistry": { - "acc": 0.2512315270935961, - "acc_stderr": 0.030516530732694436, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.3054187192118227, + "acc_stderr": 0.032406615658684086, + "brier_score": 0.7459986258062615, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_high_school_statistics": { - "acc": 0.17592592592592593, - "acc_stderr": 0.025967420958258526, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.21296296296296297, + "acc_stderr": 0.027920963147993666, + "brier_score": 0.7928295843656448, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_elementary_mathematics": { "acc": 0.24074074074074073, - "acc_stderr": 0.02201908001221789, - "timestamp": "2024-11-21T16-41-59.547447" + "acc_stderr": 0.022019080012217883, + "brier_score": 0.7598391214132458, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_electrical_engineering": { - "acc": 0.2827586206896552, - "acc_stderr": 0.03752833958003337, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.2413793103448276, + "acc_stderr": 0.03565998174135303, + "brier_score": 0.7540268437561872, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "mmlu_high_school_mathematics": { - "acc": 0.2740740740740741, - "acc_stderr": 0.027195934804085626, - "timestamp": "2024-11-21T16-41-59.547447" + "acc": 0.24074074074074073, + "acc_stderr": 0.026067159222275794, + "brier_score": 0.7597320922715378, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-53-28.313544" }, "arc_challenge": { "acc": 0.35665529010238906, diff --git a/metadata/model_scores/results_facebook__xglm-2.9B.json b/metadata/model_scores/results_facebook__xglm-2.9B.json index 937aadb..10c6be1 100644 --- a/metadata/model_scores/results_facebook__xglm-2.9B.json +++ b/metadata/model_scores/results_facebook__xglm-2.9B.json @@ -1,6 +1,6 @@ { "model_name": "facebook/xglm-2.9B", - "last_updated": "2024-12-04 11:25:08.996320", + "last_updated": "2024-12-19 13:41:13.782953", "results": { "harness": { "minerva_math_precalc": { @@ -228,289 +228,403 @@ "timestamp": "2024-06-11T12-47-19.836663" }, "mmlu_world_religions": { - "acc": 0.2631578947368421, - "acc_stderr": 0.03377310252209197, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.27485380116959063, + "acc_stderr": 0.03424042924691584, + "brier_score": 0.7784564619610387, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_formal_logic": { - "acc": 0.1746031746031746, - "acc_stderr": 0.03395490020856113, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.29365079365079366, + "acc_stderr": 0.04073524322147127, + "brier_score": 0.7845963325351285, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_prehistory": { - "acc": 0.27469135802469136, - "acc_stderr": 0.024836057868294677, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.2006172839506173, + "acc_stderr": 0.022282313949774882, + "brier_score": 0.7792786537846681, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_moral_scenarios": { - "acc": 0.2424581005586592, - "acc_stderr": 0.014333522059217887, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.23798882681564246, + "acc_stderr": 0.01424263007057489, + "brier_score": 0.7949624902702473, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_high_school_world_history": { - "acc": 0.24472573839662448, - "acc_stderr": 0.027985699387036416, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.2869198312236287, + "acc_stderr": 0.029443773022594693, + "brier_score": 0.7594604098639599, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_moral_disputes": { - "acc": 0.18208092485549132, - "acc_stderr": 0.02077676110251297, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.23410404624277456, + "acc_stderr": 0.022797110278071134, + "brier_score": 0.7971392662864681, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_professional_law": { - "acc": 0.2438070404172099, - "acc_stderr": 0.010966507972178475, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.24902216427640156, + "acc_stderr": 0.01104489226404077, + "brier_score": 0.7638688880868297, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_logical_fallacies": { - "acc": 0.27607361963190186, - "acc_stderr": 0.0351238528370505, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.24539877300613497, + "acc_stderr": 0.03380939813943354, + "brier_score": 0.8005802666734811, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_high_school_us_history": { - "acc": 0.2549019607843137, - "acc_stderr": 0.030587591351604243, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.23529411764705882, + "acc_stderr": 0.02977177522814563, + "brier_score": 0.7712167399457017, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_philosophy": { - "acc": 0.19614147909967847, - "acc_stderr": 0.022552447780478033, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.21221864951768488, + "acc_stderr": 0.023222756797435126, + "brier_score": 0.8130263770714379, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_jurisprudence": { - "acc": 0.2222222222222222, - "acc_stderr": 0.040191074725573483, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.21296296296296297, + "acc_stderr": 0.03957835471980977, + "brier_score": 0.7672667124214475, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_international_law": { - "acc": 0.35537190082644626, - "acc_stderr": 0.04369236326573981, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.24793388429752067, + "acc_stderr": 0.03941897526516304, + "brier_score": 0.7487661126399827, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_high_school_european_history": { - "acc": 0.21818181818181817, - "acc_stderr": 0.03225078108306289, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.22424242424242424, + "acc_stderr": 0.03256866661681102, + "brier_score": 0.769036899521868, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_high_school_government_and_politics": { - "acc": 0.36787564766839376, - "acc_stderr": 0.034801756684660366, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.21761658031088082, + "acc_stderr": 0.02977866303775297, + "brier_score": 0.8032216728797023, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_high_school_microeconomics": { - "acc": 0.3277310924369748, - "acc_stderr": 0.030489911417673227, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.21428571428571427, + "acc_stderr": 0.026653531596715487, + "brier_score": 0.7999394859814347, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_high_school_geography": { - "acc": 0.2474747474747475, - "acc_stderr": 0.030746300742124515, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.15151515151515152, + "acc_stderr": 0.02554565042660363, + "brier_score": 0.8162920169241648, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_high_school_psychology": { - "acc": 0.22385321100917432, - "acc_stderr": 0.017871217767790226, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.22568807339449543, + "acc_stderr": 0.017923087667803053, + "brier_score": 0.8221072465700741, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_public_relations": { - "acc": 0.23636363636363636, - "acc_stderr": 0.040693063197213754, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.2909090909090909, + "acc_stderr": 0.04350271442923243, + "brier_score": 0.7705212934293936, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_us_foreign_policy": { - "acc": 0.27, - "acc_stderr": 0.044619604333847394, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.26, + "acc_stderr": 0.0440844002276808, + "brier_score": 0.7728647597197729, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_sociology": { - "acc": 0.16417910447761194, - "acc_stderr": 0.02619392354445411, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.22388059701492538, + "acc_stderr": 0.02947525023601721, + "brier_score": 0.8034163207123037, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_high_school_macroeconomics": { - "acc": 0.2923076923076923, - "acc_stderr": 0.023060438380857747, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.22564102564102564, + "acc_stderr": 0.021193632525148526, + "brier_score": 0.7978869228998441, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_security_studies": { - "acc": 0.2163265306122449, - "acc_stderr": 0.026358916334904035, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.22040816326530613, + "acc_stderr": 0.026537045312145287, + "brier_score": 0.7820564396674529, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_professional_psychology": { - "acc": 0.2434640522875817, - "acc_stderr": 0.017362473762146644, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.24509803921568626, + "acc_stderr": 0.01740181671142766, + "brier_score": 0.7721913891320545, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_human_sexuality": { - "acc": 0.2595419847328244, - "acc_stderr": 0.03844876139785271, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.22900763358778625, + "acc_stderr": 0.036853466317118506, + "brier_score": 0.8245800855587376, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_econometrics": { - "acc": 0.2543859649122807, - "acc_stderr": 0.040969851398436716, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.2631578947368421, + "acc_stderr": 0.0414243971948936, + "brier_score": 0.7891874971383447, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_miscellaneous": { - "acc": 0.2707535121328225, - "acc_stderr": 0.01588988836256049, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.25287356321839083, + "acc_stderr": 0.015543377313719681, + "brier_score": 0.7815964758465793, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_marketing": { - "acc": 0.17094017094017094, - "acc_stderr": 0.024662496845209807, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.27350427350427353, + "acc_stderr": 0.02920254015343117, + "brier_score": 0.7637553614562805, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_management": { - "acc": 0.18446601941747573, - "acc_stderr": 0.03840423627288276, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.2524271844660194, + "acc_stderr": 0.043012503996908764, + "brier_score": 0.8082520540534411, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_nutrition": { - "acc": 0.24509803921568626, - "acc_stderr": 0.024630048979824775, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.23202614379084968, + "acc_stderr": 0.024170840879341016, + "brier_score": 0.8088553597902366, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_medical_genetics": { - "acc": 0.3, - "acc_stderr": 0.046056618647183814, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.32, + "acc_stderr": 0.046882617226215034, + "brier_score": 0.7579121532032091, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_human_aging": { - "acc": 0.23766816143497757, - "acc_stderr": 0.028568079464714274, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.32286995515695066, + "acc_stderr": 0.03138147637575498, + "brier_score": 0.7320622941617274, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_professional_medicine": { - "acc": 0.31985294117647056, - "acc_stderr": 0.028332959514031236, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.15441176470588236, + "acc_stderr": 0.021950024722922023, + "brier_score": 0.8381628978497615, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_college_medicine": { - "acc": 0.23699421965317918, - "acc_stderr": 0.03242414757483098, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.1907514450867052, + "acc_stderr": 0.029957851329869327, + "brier_score": 0.8160227156496334, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_business_ethics": { - "acc": 0.19, - "acc_stderr": 0.03942772444036623, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "brier_score": 0.7699680125818698, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_clinical_knowledge": { - "acc": 0.21509433962264152, - "acc_stderr": 0.02528839450289137, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.20754716981132076, + "acc_stderr": 0.024959918028911267, + "brier_score": 0.8024709883934613, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_global_facts": { - "acc": 0.18, - "acc_stderr": 0.038612291966536955, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.23, + "acc_stderr": 0.042295258468165065, + "brier_score": 0.7755939405534948, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_virology": { - "acc": 0.2289156626506024, - "acc_stderr": 0.03270745277352477, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.2891566265060241, + "acc_stderr": 0.03529486801511115, + "brier_score": 0.7675743285611036, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_professional_accounting": { - "acc": 0.22695035460992907, - "acc_stderr": 0.024987106365642962, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.2765957446808511, + "acc_stderr": 0.026684564340460983, + "brier_score": 0.7612596779770814, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_college_physics": { - "acc": 0.23529411764705882, - "acc_stderr": 0.04220773659171452, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.20588235294117646, + "acc_stderr": 0.04023382273617747, + "brier_score": 0.8147810206949243, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_high_school_physics": { - "acc": 0.304635761589404, - "acc_stderr": 0.03757949922943342, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.2052980132450331, + "acc_stderr": 0.03297986648473834, + "brier_score": 0.7850270619906705, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_high_school_biology": { - "acc": 0.31290322580645163, - "acc_stderr": 0.02637756702864586, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.2032258064516129, + "acc_stderr": 0.02289168798455495, + "brier_score": 0.7879831869554097, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_college_biology": { - "acc": 0.2708333333333333, - "acc_stderr": 0.03716177437566016, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.2569444444444444, + "acc_stderr": 0.03653946969442099, + "brier_score": 0.7667620151908258, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_anatomy": { - "acc": 0.32592592592592595, - "acc_stderr": 0.040491220417025055, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.1925925925925926, + "acc_stderr": 0.03406542058502653, + "brier_score": 0.8159042810752004, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_college_chemistry": { - "acc": 0.23, - "acc_stderr": 0.042295258468165065, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.18, + "acc_stderr": 0.03861229196653696, + "brier_score": 0.8232613704210737, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_computer_security": { - "acc": 0.16, - "acc_stderr": 0.0368452949177471, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.29, + "acc_stderr": 0.04560480215720684, + "brier_score": 0.7810317638069373, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_college_computer_science": { - "acc": 0.24, - "acc_stderr": 0.04292346959909283, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.23, + "acc_stderr": 0.04229525846816506, + "brier_score": 0.7978820652141898, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_astronomy": { - "acc": 0.17763157894736842, - "acc_stderr": 0.031103182383123398, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.18421052631578946, + "acc_stderr": 0.0315469804508223, + "brier_score": 0.8121393101079877, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_college_mathematics": { - "acc": 0.24, - "acc_stderr": 0.04292346959909281, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.22, + "acc_stderr": 0.0416333199893227, + "brier_score": 0.8056959732869093, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_conceptual_physics": { - "acc": 0.2297872340425532, - "acc_stderr": 0.027501752944412424, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.26382978723404255, + "acc_stderr": 0.028809989854102977, + "brier_score": 0.7880471540031094, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_abstract_algebra": { - "acc": 0.27, - "acc_stderr": 0.044619604333847394, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.22, + "acc_stderr": 0.04163331998932269, + "brier_score": 0.8258750816843782, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_high_school_computer_science": { - "acc": 0.32, - "acc_stderr": 0.04688261722621504, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.29, + "acc_stderr": 0.045604802157206845, + "brier_score": 0.7564390065224648, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_machine_learning": { - "acc": 0.29464285714285715, - "acc_stderr": 0.043270409325787317, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.3392857142857143, + "acc_stderr": 0.04493949068613539, + "brier_score": 0.7458255697862292, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_high_school_chemistry": { - "acc": 0.3054187192118227, - "acc_stderr": 0.032406615658684086, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.1921182266009852, + "acc_stderr": 0.027719315709614778, + "brier_score": 0.812417944584597, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_high_school_statistics": { - "acc": 0.4675925925925926, - "acc_stderr": 0.03402801581358966, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.17592592592592593, + "acc_stderr": 0.025967420958258526, + "brier_score": 0.7912250369489995, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_elementary_mathematics": { - "acc": 0.2619047619047619, - "acc_stderr": 0.022644212615525218, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.2566137566137566, + "acc_stderr": 0.02249451076750315, + "brier_score": 0.7938783769148282, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_electrical_engineering": { - "acc": 0.2689655172413793, - "acc_stderr": 0.036951833116502325, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.2413793103448276, + "acc_stderr": 0.03565998174135302, + "brier_score": 0.8314571330920377, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "mmlu_high_school_mathematics": { - "acc": 0.29259259259259257, - "acc_stderr": 0.02773896963217609, - "timestamp": "2024-11-20T02-09-39.233954" + "acc": 0.24444444444444444, + "acc_stderr": 0.02620276653465215, + "brier_score": 0.7939871511609232, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-17.950634" }, "arc_challenge": { "acc": 0.2764505119453925, diff --git a/metadata/model_scores/results_facebook__xglm-4.5B.json b/metadata/model_scores/results_facebook__xglm-4.5B.json index c2cbd41..c89b87d 100644 --- a/metadata/model_scores/results_facebook__xglm-4.5B.json +++ b/metadata/model_scores/results_facebook__xglm-4.5B.json @@ -1,6 +1,6 @@ { "model_name": "facebook/xglm-4.5B", - "last_updated": "2024-12-04 11:23:11.491681", + "last_updated": "2024-12-19 13:38:27.656223", "results": { "harness": { "minerva_math_precalc": { @@ -228,289 +228,403 @@ "timestamp": "2024-06-09T15-48-16.680733" }, "mmlu_world_religions": { - "acc": 0.21052631578947367, - "acc_stderr": 0.031267817146631786, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.27485380116959063, + "acc_stderr": 0.034240429246915824, + "brier_score": 0.755274678758315, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_formal_logic": { - "acc": 0.1746031746031746, - "acc_stderr": 0.033954900208561116, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.2777777777777778, + "acc_stderr": 0.04006168083848876, + "brier_score": 0.7663550040673333, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_prehistory": { - "acc": 0.2654320987654321, - "acc_stderr": 0.024569223600460845, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.24074074074074073, + "acc_stderr": 0.023788583551658533, + "brier_score": 0.7652663553372573, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_moral_scenarios": { - "acc": 0.23798882681564246, - "acc_stderr": 0.01424263007057487, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.24581005586592178, + "acc_stderr": 0.014400296429225622, + "brier_score": 0.7672549005501607, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_high_school_world_history": { - "acc": 0.2320675105485232, - "acc_stderr": 0.027479744550808514, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.270042194092827, + "acc_stderr": 0.028900721906293426, + "brier_score": 0.7571285708105355, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_moral_disputes": { - "acc": 0.26878612716763006, - "acc_stderr": 0.023868003262500107, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.23699421965317918, + "acc_stderr": 0.02289408248992599, + "brier_score": 0.7647857763853563, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_professional_law": { - "acc": 0.26988265971316816, - "acc_stderr": 0.011337381084250416, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.2607561929595828, + "acc_stderr": 0.011213471559602338, + "brier_score": 0.7557701064945394, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_logical_fallacies": { - "acc": 0.3006134969325153, - "acc_stderr": 0.03602511318806771, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.26993865030674846, + "acc_stderr": 0.034878251684978906, + "brier_score": 0.7551927191767064, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_high_school_us_history": { - "acc": 0.2107843137254902, - "acc_stderr": 0.028626547912437378, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.24019607843137256, + "acc_stderr": 0.02998373305591362, + "brier_score": 0.7678181155256781, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_philosophy": { - "acc": 0.3022508038585209, - "acc_stderr": 0.026082700695399662, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.2057877813504823, + "acc_stderr": 0.02296133990676424, + "brier_score": 0.7916388053551179, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_jurisprudence": { - "acc": 0.23148148148148148, - "acc_stderr": 0.04077494709252626, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.2962962962962963, + "acc_stderr": 0.044143436668549335, + "brier_score": 0.7521606226757167, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_international_law": { - "acc": 0.34710743801652894, - "acc_stderr": 0.04345724570292534, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.2644628099173554, + "acc_stderr": 0.04026187527591207, + "brier_score": 0.7654804359244274, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_high_school_european_history": { - "acc": 0.21212121212121213, - "acc_stderr": 0.031922715695482995, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.20606060606060606, + "acc_stderr": 0.03158415324047709, + "brier_score": 0.7862340210921961, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_high_school_government_and_politics": { - "acc": 0.2538860103626943, - "acc_stderr": 0.03141024780565317, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.22797927461139897, + "acc_stderr": 0.030276909945178267, + "brier_score": 0.7665234321742033, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_high_school_microeconomics": { - "acc": 0.23529411764705882, - "acc_stderr": 0.027553614467863786, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.226890756302521, + "acc_stderr": 0.027205371538279476, + "brier_score": 0.7561418553252524, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_high_school_geography": { - "acc": 0.2676767676767677, - "acc_stderr": 0.03154449888270286, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.18686868686868688, + "acc_stderr": 0.02777253333421899, + "brier_score": 0.7781380168315729, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_high_school_psychology": { - "acc": 0.25504587155963304, - "acc_stderr": 0.018688500856535836, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.21100917431192662, + "acc_stderr": 0.017493922404112648, + "brier_score": 0.7756897923681646, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_public_relations": { - "acc": 0.23636363636363636, - "acc_stderr": 0.04069306319721377, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.3, + "acc_stderr": 0.04389311454644287, + "brier_score": 0.744357285811644, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_us_foreign_policy": { - "acc": 0.28, - "acc_stderr": 0.04512608598542127, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.24, + "acc_stderr": 0.04292346959909282, + "brier_score": 0.7615163811611504, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_sociology": { - "acc": 0.27860696517412936, - "acc_stderr": 0.0317005618349731, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.21890547263681592, + "acc_stderr": 0.029239174636647, + "brier_score": 0.7675237295477422, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_high_school_macroeconomics": { - "acc": 0.24358974358974358, - "acc_stderr": 0.021763733684173937, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.24102564102564103, + "acc_stderr": 0.021685546665333195, + "brier_score": 0.7628253979700401, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_security_studies": { - "acc": 0.23673469387755103, - "acc_stderr": 0.027212835884073153, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.2897959183673469, + "acc_stderr": 0.029043088683304324, + "brier_score": 0.7478338865347484, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_professional_psychology": { - "acc": 0.2565359477124183, - "acc_stderr": 0.017667841612379, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.25980392156862747, + "acc_stderr": 0.017740899509177788, + "brier_score": 0.7549298076413554, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_human_sexuality": { - "acc": 0.26717557251908397, - "acc_stderr": 0.03880848301082397, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.22137404580152673, + "acc_stderr": 0.03641297081313729, + "brier_score": 0.7843585183451098, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_econometrics": { - "acc": 0.2719298245614035, - "acc_stderr": 0.041857744240220554, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.24561403508771928, + "acc_stderr": 0.04049339297748141, + "brier_score": 0.7519263840177439, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_miscellaneous": { - "acc": 0.20689655172413793, - "acc_stderr": 0.014485656041669173, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.2247765006385696, + "acc_stderr": 0.014927447101937164, + "brier_score": 0.7688507326782054, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_marketing": { - "acc": 0.23931623931623933, - "acc_stderr": 0.027951826808924333, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.23076923076923078, + "acc_stderr": 0.027601921381417628, + "brier_score": 0.759167155406607, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_management": { - "acc": 0.2912621359223301, - "acc_stderr": 0.044986763205729224, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.21359223300970873, + "acc_stderr": 0.04058042015646034, + "brier_score": 0.7666997072926246, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_nutrition": { - "acc": 0.2581699346405229, - "acc_stderr": 0.025058503316958147, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.2647058823529412, + "acc_stderr": 0.025261691219729484, + "brier_score": 0.7684122808652835, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_medical_genetics": { - "acc": 0.26, - "acc_stderr": 0.04408440022768078, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.32, + "acc_stderr": 0.04688261722621505, + "brier_score": 0.7475889299146562, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_human_aging": { - "acc": 0.21524663677130046, - "acc_stderr": 0.027584066602208274, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.336322869955157, + "acc_stderr": 0.031708824268455005, + "brier_score": 0.7285624927346905, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_professional_medicine": { - "acc": 0.3272058823529412, - "acc_stderr": 0.02850145286039657, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.1875, + "acc_stderr": 0.023709788253811766, + "brier_score": 0.7905421043154426, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_college_medicine": { - "acc": 0.27167630057803466, - "acc_stderr": 0.03391750322321659, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.23121387283236994, + "acc_stderr": 0.032147373020294696, + "brier_score": 0.7691126269486598, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_business_ethics": { - "acc": 0.23, - "acc_stderr": 0.04229525846816505, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.32, + "acc_stderr": 0.046882617226215034, + "brier_score": 0.756343264035676, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_clinical_knowledge": { - "acc": 0.22641509433962265, - "acc_stderr": 0.025757559893106727, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.19622641509433963, + "acc_stderr": 0.024442388131100824, + "brier_score": 0.77492420767386, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_global_facts": { - "acc": 0.34, - "acc_stderr": 0.047609522856952344, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.19, + "acc_stderr": 0.039427724440366234, + "brier_score": 0.7720125546673212, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_virology": { - "acc": 0.1927710843373494, - "acc_stderr": 0.03070982405056527, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.27710843373493976, + "acc_stderr": 0.03484331592680588, + "brier_score": 0.752677142414452, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_professional_accounting": { - "acc": 0.2765957446808511, - "acc_stderr": 0.026684564340460997, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.2872340425531915, + "acc_stderr": 0.026992199173064356, + "brier_score": 0.7521154419726764, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_college_physics": { - "acc": 0.19607843137254902, - "acc_stderr": 0.03950581861179962, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.3235294117647059, + "acc_stderr": 0.046550104113196177, + "brier_score": 0.7634458806639429, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_high_school_physics": { - "acc": 0.26490066225165565, - "acc_stderr": 0.03603038545360385, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.2185430463576159, + "acc_stderr": 0.03374235550425694, + "brier_score": 0.7694405034154213, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_high_school_biology": { - "acc": 0.2645161290322581, - "acc_stderr": 0.02509189237885928, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.2709677419354839, + "acc_stderr": 0.025284416114900156, + "brier_score": 0.7571428679121684, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_college_biology": { - "acc": 0.2708333333333333, - "acc_stderr": 0.037161774375660164, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.2986111111111111, + "acc_stderr": 0.03827052357950756, + "brier_score": 0.7522643021022564, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_anatomy": { - "acc": 0.32592592592592595, - "acc_stderr": 0.040491220417025055, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.2814814814814815, + "acc_stderr": 0.03885004245800254, + "brier_score": 0.7575248956377039, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_college_chemistry": { - "acc": 0.24, - "acc_stderr": 0.04292346959909283, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "brier_score": 0.784057389814469, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_computer_security": { - "acc": 0.29, - "acc_stderr": 0.04560480215720684, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 0.7511719218716887, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_college_computer_science": { - "acc": 0.28, - "acc_stderr": 0.04512608598542128, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.29, + "acc_stderr": 0.04560480215720684, + "brier_score": 0.7568992421562741, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_astronomy": { - "acc": 0.2236842105263158, - "acc_stderr": 0.033911609343436025, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.21052631578947367, + "acc_stderr": 0.03317672787533157, + "brier_score": 0.7923962788237392, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_college_mathematics": { - "acc": 0.28, - "acc_stderr": 0.045126085985421296, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 0.7616331652332431, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_conceptual_physics": { - "acc": 0.2425531914893617, - "acc_stderr": 0.028020226271200217, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.20425531914893616, + "acc_stderr": 0.026355158413349424, + "brier_score": 0.7668170378959265, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_abstract_algebra": { - "acc": 0.23, - "acc_stderr": 0.04229525846816505, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 0.7643398455539884, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_high_school_computer_science": { - "acc": 0.29, - "acc_stderr": 0.045604802157206845, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.28, + "acc_stderr": 0.045126085985421276, + "brier_score": 0.7451260614096664, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_machine_learning": { - "acc": 0.25, - "acc_stderr": 0.04109974682633932, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.2767857142857143, + "acc_stderr": 0.042466243366976235, + "brier_score": 0.7638119478953422, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_high_school_chemistry": { "acc": 0.24630541871921183, - "acc_stderr": 0.030315099285617722, - "timestamp": "2024-11-10T14-16-29.255958" + "acc_stderr": 0.03031509928561773, + "brier_score": 0.7686052199936642, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_high_school_statistics": { - "acc": 0.2777777777777778, - "acc_stderr": 0.03054674526495319, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.2916666666666667, + "acc_stderr": 0.030998666304560524, + "brier_score": 0.7430103334400534, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_elementary_mathematics": { - "acc": 0.25396825396825395, - "acc_stderr": 0.022418042891113946, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.2222222222222222, + "acc_stderr": 0.021411684393694196, + "brier_score": 0.7801099705145796, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_electrical_engineering": { "acc": 0.23448275862068965, "acc_stderr": 0.035306258743465914, - "timestamp": "2024-11-10T14-16-29.255958" + "brier_score": 0.7613993454942953, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "mmlu_high_school_mathematics": { - "acc": 0.24814814814814815, - "acc_stderr": 0.0263357394040558, - "timestamp": "2024-11-10T14-16-29.255958" + "acc": 0.22592592592592592, + "acc_stderr": 0.02549753263960955, + "brier_score": 0.7656972806536992, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-43.759015" }, "arc_challenge": { "acc": 0.2977815699658703, diff --git a/metadata/model_scores/results_facebook__xglm-564M.json b/metadata/model_scores/results_facebook__xglm-564M.json index cdd0b6a..74b3672 100644 --- a/metadata/model_scores/results_facebook__xglm-564M.json +++ b/metadata/model_scores/results_facebook__xglm-564M.json @@ -1,6 +1,6 @@ { "model_name": "facebook/xglm-564M", - "last_updated": "2024-12-04 11:22:45.502176", + "last_updated": "2024-12-19 13:38:05.443940", "results": { "harness": { "minerva_math_precalc": { @@ -228,289 +228,403 @@ "timestamp": "2024-06-09T15-17-16.795144" }, "mmlu_world_religions": { - "acc": 0.22807017543859648, - "acc_stderr": 0.032180937956023566, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.3216374269005848, + "acc_stderr": 0.03582529442573122, + "brier_score": 1.0742204944517322, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_formal_logic": { - "acc": 0.14285714285714285, - "acc_stderr": 0.03129843185743809, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.29365079365079366, + "acc_stderr": 0.04073524322147127, + "brier_score": 0.9756313169804313, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_prehistory": { - "acc": 0.21604938271604937, - "acc_stderr": 0.02289916291844581, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.21296296296296297, + "acc_stderr": 0.022779719088733396, + "brier_score": 1.1572943708648094, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_moral_scenarios": { - "acc": 0.2424581005586592, - "acc_stderr": 0.014333522059217892, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.23798882681564246, + "acc_stderr": 0.014242630070574885, + "brier_score": 1.0254183250723512, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_high_school_world_history": { - "acc": 0.2489451476793249, - "acc_stderr": 0.028146970599422644, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.270042194092827, + "acc_stderr": 0.028900721906293426, + "brier_score": 1.018241284346488, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_moral_disputes": { "acc": 0.24855491329479767, "acc_stderr": 0.023267528432100174, - "timestamp": "2024-11-22T20-36-49.643550" + "brier_score": 0.9919887099294075, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_professional_law": { "acc": 0.2457627118644068, "acc_stderr": 0.01099615663514269, - "timestamp": "2024-11-22T20-36-49.643550" + "brier_score": 1.14760371837571, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_logical_fallacies": { - "acc": 0.26380368098159507, - "acc_stderr": 0.03462419931615624, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.22085889570552147, + "acc_stderr": 0.032591773927421776, + "brier_score": 1.1609613230884086, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_high_school_us_history": { - "acc": 0.27941176470588236, - "acc_stderr": 0.031493281045079556, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.25, + "acc_stderr": 0.03039153369274154, + "brier_score": 1.0642376995987706, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_philosophy": { - "acc": 0.1832797427652733, - "acc_stderr": 0.021974198848265812, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.1864951768488746, + "acc_stderr": 0.02212243977248077, + "brier_score": 1.0991696264667123, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_jurisprudence": { "acc": 0.25925925925925924, - "acc_stderr": 0.04236511258094631, - "timestamp": "2024-11-22T20-36-49.643550" + "acc_stderr": 0.04236511258094634, + "brier_score": 1.1133413634160059, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_international_law": { - "acc": 0.32231404958677684, - "acc_stderr": 0.042664163633521664, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.2396694214876033, + "acc_stderr": 0.03896878985070417, + "brier_score": 1.1945862952663897, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_high_school_european_history": { - "acc": 0.26666666666666666, - "acc_stderr": 0.03453131801885415, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.21818181818181817, + "acc_stderr": 0.03225078108306289, + "brier_score": 1.0930421099365542, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_high_school_government_and_politics": { - "acc": 0.2694300518134715, - "acc_stderr": 0.03201867122877794, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.19689119170984457, + "acc_stderr": 0.02869787397186069, + "brier_score": 1.2726744501976759, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_high_school_microeconomics": { - "acc": 0.23529411764705882, - "acc_stderr": 0.02755361446786379, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.21008403361344538, + "acc_stderr": 0.026461398717471874, + "brier_score": 1.1642584812060601, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_high_school_geography": { - "acc": 0.18686868686868688, - "acc_stderr": 0.02777253333421898, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.17676767676767677, + "acc_stderr": 0.027178752639044915, + "brier_score": 1.2243462537350878, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_high_school_psychology": { "acc": 0.1926605504587156, - "acc_stderr": 0.016909276884936087, - "timestamp": "2024-11-22T20-36-49.643550" + "acc_stderr": 0.016909276884936073, + "brier_score": 1.1864906805081876, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_public_relations": { "acc": 0.21818181818181817, "acc_stderr": 0.03955932861795833, - "timestamp": "2024-11-22T20-36-49.643550" + "brier_score": 1.171621547436488, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_us_foreign_policy": { - "acc": 0.23, - "acc_stderr": 0.04229525846816507, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.28, + "acc_stderr": 0.045126085985421276, + "brier_score": 1.041866918047384, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_sociology": { - "acc": 0.22885572139303484, - "acc_stderr": 0.029705284056772436, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.24378109452736318, + "acc_stderr": 0.030360490154014652, + "brier_score": 1.1864462264687594, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_high_school_macroeconomics": { - "acc": 0.2076923076923077, - "acc_stderr": 0.0205675395672468, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.20256410256410257, + "acc_stderr": 0.020377660970371397, + "brier_score": 1.144645957965444, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_security_studies": { - "acc": 0.2612244897959184, - "acc_stderr": 0.028123429335142787, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.1836734693877551, + "acc_stderr": 0.024789071332007643, + "brier_score": 1.0750865427602068, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_professional_psychology": { - "acc": 0.25, - "acc_stderr": 0.01751781884501444, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.25326797385620914, + "acc_stderr": 0.017593486895366835, + "brier_score": 1.10367136949964, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_human_sexuality": { "acc": 0.2595419847328244, "acc_stderr": 0.03844876139785271, - "timestamp": "2024-11-22T20-36-49.643550" + "brier_score": 1.1555577803376484, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_econometrics": { - "acc": 0.2719298245614035, - "acc_stderr": 0.04185774424022056, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.23684210526315788, + "acc_stderr": 0.039994238792813386, + "brier_score": 1.1043201571671586, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_miscellaneous": { "acc": 0.23754789272030652, - "acc_stderr": 0.015218733046150191, - "timestamp": "2024-11-22T20-36-49.643550" + "acc_stderr": 0.015218733046150195, + "brier_score": 1.1747437386100705, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_marketing": { - "acc": 0.2948717948717949, - "acc_stderr": 0.02987257770889115, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.2905982905982906, + "acc_stderr": 0.029745048572674057, + "brier_score": 1.017501207234622, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_management": { - "acc": 0.18446601941747573, - "acc_stderr": 0.03840423627288276, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.17475728155339806, + "acc_stderr": 0.03760178006026621, + "brier_score": 1.3234052284173319, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_nutrition": { - "acc": 0.21895424836601307, - "acc_stderr": 0.02367908986180772, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.22549019607843138, + "acc_stderr": 0.023929155517351284, + "brier_score": 1.0904194849137472, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_medical_genetics": { "acc": 0.3, "acc_stderr": 0.046056618647183814, - "timestamp": "2024-11-22T20-36-49.643550" + "brier_score": 1.105144331505844, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_human_aging": { - "acc": 0.33183856502242154, - "acc_stderr": 0.03160295143776679, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.3094170403587444, + "acc_stderr": 0.031024411740572206, + "brier_score": 1.0500046831441932, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_professional_medicine": { - "acc": 0.44485294117647056, - "acc_stderr": 0.030187532060329383, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.18382352941176472, + "acc_stderr": 0.02352924218519311, + "brier_score": 1.3024801814973317, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_college_medicine": { - "acc": 0.2138728323699422, - "acc_stderr": 0.03126511206173042, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.20809248554913296, + "acc_stderr": 0.030952890217749884, + "brier_score": 1.127110928046614, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_business_ethics": { - "acc": 0.28, - "acc_stderr": 0.045126085985421296, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "brier_score": 0.9902005073726969, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_clinical_knowledge": { - "acc": 0.2037735849056604, - "acc_stderr": 0.024790784501775406, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.21509433962264152, + "acc_stderr": 0.025288394502891377, + "brier_score": 1.1457897742784553, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_global_facts": { "acc": 0.18, - "acc_stderr": 0.03861229196653694, - "timestamp": "2024-11-22T20-36-49.643550" + "acc_stderr": 0.038612291966536955, + "brier_score": 1.2337053484077747, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_virology": { "acc": 0.28313253012048195, - "acc_stderr": 0.03507295431370519, - "timestamp": "2024-11-22T20-36-49.643550" + "acc_stderr": 0.03507295431370518, + "brier_score": 1.0801259462709316, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_professional_accounting": { - "acc": 0.2801418439716312, - "acc_stderr": 0.026789172351140245, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.23404255319148937, + "acc_stderr": 0.025257861359432407, + "brier_score": 1.1444845677900901, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_college_physics": { - "acc": 0.23529411764705882, - "acc_stderr": 0.04220773659171453, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.21568627450980393, + "acc_stderr": 0.040925639582376556, + "brier_score": 1.254457287294448, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_high_school_physics": { - "acc": 0.2052980132450331, - "acc_stderr": 0.03297986648473835, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.1986754966887417, + "acc_stderr": 0.032578473844367746, + "brier_score": 1.2603638914713362, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_high_school_biology": { - "acc": 0.2709677419354839, - "acc_stderr": 0.02528441611490016, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.1774193548387097, + "acc_stderr": 0.021732540689329265, + "brier_score": 1.1962168644512337, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_college_biology": { - "acc": 0.2222222222222222, - "acc_stderr": 0.03476590104304134, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.2638888888888889, + "acc_stderr": 0.03685651095897532, + "brier_score": 1.10328888433029, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_anatomy": { - "acc": 0.2962962962962963, - "acc_stderr": 0.03944624162501116, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.18518518518518517, + "acc_stderr": 0.03355677216313142, + "brier_score": 1.2379006196342983, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_college_chemistry": { "acc": 0.2, - "acc_stderr": 0.04020151261036845, - "timestamp": "2024-11-22T20-36-49.643550" + "acc_stderr": 0.040201512610368445, + "brier_score": 1.1775779464579272, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_computer_security": { - "acc": 0.25, - "acc_stderr": 0.04351941398892446, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.28, + "acc_stderr": 0.045126085985421276, + "brier_score": 1.0988299111046593, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_college_computer_science": { - "acc": 0.16, - "acc_stderr": 0.036845294917747094, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 1.1186786193930078, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_astronomy": { "acc": 0.17763157894736842, - "acc_stderr": 0.031103182383123387, - "timestamp": "2024-11-22T20-36-49.643550" + "acc_stderr": 0.031103182383123398, + "brier_score": 1.2905734218280234, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_college_mathematics": { - "acc": 0.24, - "acc_stderr": 0.04292346959909282, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "brier_score": 1.192045879915838, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_conceptual_physics": { - "acc": 0.30638297872340425, - "acc_stderr": 0.03013590647851756, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.26382978723404255, + "acc_stderr": 0.02880998985410298, + "brier_score": 1.06926722842903, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_abstract_algebra": { - "acc": 0.2, - "acc_stderr": 0.040201512610368445, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.22, + "acc_stderr": 0.04163331998932269, + "brier_score": 1.1232698972017345, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_high_school_computer_science": { - "acc": 0.31, - "acc_stderr": 0.046482319871173156, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 1.1409648640225867, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_machine_learning": { - "acc": 0.3392857142857143, - "acc_stderr": 0.04493949068613539, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.3125, + "acc_stderr": 0.043994650575715215, + "brier_score": 0.9704242816152965, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_high_school_chemistry": { - "acc": 0.2955665024630542, - "acc_stderr": 0.032104944337514575, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.15270935960591134, + "acc_stderr": 0.025308904539380624, + "brier_score": 1.197587214007225, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_high_school_statistics": { - "acc": 0.4583333333333333, - "acc_stderr": 0.03398110890294636, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.1527777777777778, + "acc_stderr": 0.02453632602613422, + "brier_score": 1.3412212646761488, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_elementary_mathematics": { - "acc": 0.2566137566137566, - "acc_stderr": 0.022494510767503154, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.20899470899470898, + "acc_stderr": 0.020940481565334835, + "brier_score": 1.2361802328965794, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_electrical_engineering": { - "acc": 0.25517241379310346, - "acc_stderr": 0.03632984052707842, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.2413793103448276, + "acc_stderr": 0.03565998174135302, + "brier_score": 1.1751296553870207, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "mmlu_high_school_mathematics": { - "acc": 0.23703703703703705, - "acc_stderr": 0.02592887613276614, - "timestamp": "2024-11-22T20-36-49.643550" + "acc": 0.2111111111111111, + "acc_stderr": 0.02488211685765508, + "brier_score": 1.249433087090846, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-21-59.412791" }, "arc_challenge": { "acc": 0.20648464163822525, diff --git a/metadata/model_scores/results_facebook__xglm-7.5B.json b/metadata/model_scores/results_facebook__xglm-7.5B.json index 68777d2..993545a 100644 --- a/metadata/model_scores/results_facebook__xglm-7.5B.json +++ b/metadata/model_scores/results_facebook__xglm-7.5B.json @@ -1,6 +1,6 @@ { "model_name": "facebook/xglm-7.5B", - "last_updated": "2024-12-04 11:22:39.577336", + "last_updated": "2024-12-19 13:37:56.869427", "results": { "harness": { "minerva_math_precalc": { @@ -228,289 +228,403 @@ "timestamp": "2024-06-09T16-01-07.820079" }, "mmlu_world_religions": { - "acc": 0.27485380116959063, - "acc_stderr": 0.03424042924691583, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.3157894736842105, + "acc_stderr": 0.03565079670708312, + "brier_score": 0.7478836978346121, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_formal_logic": { - "acc": 0.31746031746031744, - "acc_stderr": 0.04163453031302859, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.21428571428571427, + "acc_stderr": 0.03670066451047181, + "brier_score": 0.7711352778112898, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_prehistory": { - "acc": 0.2716049382716049, - "acc_stderr": 0.024748624490537365, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.25617283950617287, + "acc_stderr": 0.0242885336377261, + "brier_score": 0.7602271097108688, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_moral_scenarios": { - "acc": 0.2424581005586592, - "acc_stderr": 0.014333522059217892, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.23687150837988827, + "acc_stderr": 0.01421957078810398, + "brier_score": 0.7704458434686978, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_high_school_world_history": { - "acc": 0.25316455696202533, - "acc_stderr": 0.028304657943035296, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.24472573839662448, + "acc_stderr": 0.02798569938703641, + "brier_score": 0.7612080943321962, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_moral_disputes": { - "acc": 0.23121387283236994, - "acc_stderr": 0.02269865716785571, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.24277456647398843, + "acc_stderr": 0.023083658586984204, + "brier_score": 0.7620153279156361, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_professional_law": { - "acc": 0.2438070404172099, - "acc_stderr": 0.010966507972178475, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.2457627118644068, + "acc_stderr": 0.010996156635142692, + "brier_score": 0.7593523563741399, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_logical_fallacies": { - "acc": 0.25153374233128833, - "acc_stderr": 0.03408997886857529, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.26993865030674846, + "acc_stderr": 0.034878251684978906, + "brier_score": 0.7643975216702128, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_high_school_us_history": { - "acc": 0.23529411764705882, - "acc_stderr": 0.029771775228145628, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.2107843137254902, + "acc_stderr": 0.0286265479124374, + "brier_score": 0.7736428893138484, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_philosophy": { - "acc": 0.2315112540192926, - "acc_stderr": 0.023956532766639137, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.27009646302250806, + "acc_stderr": 0.025218040373410616, + "brier_score": 0.7652509223419405, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_jurisprudence": { - "acc": 0.21296296296296297, - "acc_stderr": 0.03957835471980979, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.28703703703703703, + "acc_stderr": 0.04373313040914761, + "brier_score": 0.7492958135151253, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_international_law": { - "acc": 0.34710743801652894, - "acc_stderr": 0.043457245702925335, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.256198347107438, + "acc_stderr": 0.03984979653302871, + "brier_score": 0.7704808114213275, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_high_school_european_history": { - "acc": 0.28484848484848485, - "acc_stderr": 0.03524390844511783, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.20606060606060606, + "acc_stderr": 0.03158415324047708, + "brier_score": 0.7849059442327416, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_high_school_government_and_politics": { - "acc": 0.3626943005181347, - "acc_stderr": 0.034697137917043715, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.23834196891191708, + "acc_stderr": 0.03074890536390989, + "brier_score": 0.7804161088743681, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_high_school_microeconomics": { - "acc": 0.3319327731092437, - "acc_stderr": 0.030588697013783663, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.25630252100840334, + "acc_stderr": 0.02835962087053395, + "brier_score": 0.7618685447381729, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_high_school_geography": { - "acc": 0.36363636363636365, - "acc_stderr": 0.03427308652999933, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.1919191919191919, + "acc_stderr": 0.02805779167298901, + "brier_score": 0.7856532168185606, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_high_school_psychology": { - "acc": 0.26238532110091745, - "acc_stderr": 0.018861885021534738, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.21100917431192662, + "acc_stderr": 0.01749392240411265, + "brier_score": 0.7825017004964188, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_public_relations": { - "acc": 0.24545454545454545, - "acc_stderr": 0.041220665028782834, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.3090909090909091, + "acc_stderr": 0.044262946482000985, + "brier_score": 0.7554740125877594, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_us_foreign_policy": { - "acc": 0.22, - "acc_stderr": 0.04163331998932269, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.26, + "acc_stderr": 0.044084400227680794, + "brier_score": 0.758873841872607, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_sociology": { - "acc": 0.23880597014925373, - "acc_stderr": 0.030147775935409217, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.23383084577114427, + "acc_stderr": 0.029929415408348398, + "brier_score": 0.7768714367937326, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_high_school_macroeconomics": { - "acc": 0.3564102564102564, - "acc_stderr": 0.0242831405294673, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.2358974358974359, + "acc_stderr": 0.021525965407408726, + "brier_score": 0.763402389499434, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_security_studies": { - "acc": 0.2857142857142857, - "acc_stderr": 0.0289205832206756, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.3142857142857143, + "acc_stderr": 0.02971932942241748, + "brier_score": 0.744842735795794, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_professional_psychology": { - "acc": 0.2647058823529412, - "acc_stderr": 0.01784808957491322, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.25326797385620914, + "acc_stderr": 0.017593486895366835, + "brier_score": 0.7584474284269817, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_human_sexuality": { - "acc": 0.2366412213740458, - "acc_stderr": 0.03727673575596917, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.2748091603053435, + "acc_stderr": 0.039153454088478354, + "brier_score": 0.7664670824349369, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_econometrics": { - "acc": 0.21929824561403508, - "acc_stderr": 0.038924311065187504, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.21052631578947367, + "acc_stderr": 0.038351539543994194, + "brier_score": 0.755762572716479, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_miscellaneous": { - "acc": 0.26053639846743293, - "acc_stderr": 0.015696008563807092, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.2413793103448276, + "acc_stderr": 0.015302380123542094, + "brier_score": 0.7665063300029615, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_marketing": { - "acc": 0.18803418803418803, - "acc_stderr": 0.025598193686652244, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.26495726495726496, + "acc_stderr": 0.028911208802749458, + "brier_score": 0.7462265461625658, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_management": { - "acc": 0.2815533980582524, - "acc_stderr": 0.04453254836326466, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.23300970873786409, + "acc_stderr": 0.04185832598928315, + "brier_score": 0.757532453734775, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_nutrition": { - "acc": 0.2777777777777778, - "acc_stderr": 0.02564686309713791, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.24183006535947713, + "acc_stderr": 0.024518195641879334, + "brier_score": 0.7661987299388116, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_medical_genetics": { - "acc": 0.31, - "acc_stderr": 0.04648231987117316, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "brier_score": 0.7445708685279325, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_human_aging": { - "acc": 0.3273542600896861, - "acc_stderr": 0.03149384670994131, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.32286995515695066, + "acc_stderr": 0.03138147637575499, + "brier_score": 0.7165132661444996, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_professional_medicine": { - "acc": 0.4522058823529412, - "acc_stderr": 0.03023375855159645, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.1875, + "acc_stderr": 0.023709788253811766, + "brier_score": 0.8085025990033413, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_college_medicine": { - "acc": 0.2543352601156069, - "acc_stderr": 0.0332055644308557, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.27167630057803466, + "acc_stderr": 0.03391750322321659, + "brier_score": 0.7591816490931492, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_business_ethics": { - "acc": 0.25, - "acc_stderr": 0.04351941398892446, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.21, + "acc_stderr": 0.04093601807403326, + "brier_score": 0.7598132441980852, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_clinical_knowledge": { - "acc": 0.3018867924528302, - "acc_stderr": 0.028254200344438655, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.23773584905660378, + "acc_stderr": 0.02619980880756191, + "brier_score": 0.7633841188520737, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_global_facts": { - "acc": 0.3, - "acc_stderr": 0.046056618647183814, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.29, + "acc_stderr": 0.045604802157206845, + "brier_score": 0.7547408714872293, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_virology": { - "acc": 0.2891566265060241, - "acc_stderr": 0.035294868015111155, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.27710843373493976, + "acc_stderr": 0.034843315926805875, + "brier_score": 0.7484904151371046, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_professional_accounting": { - "acc": 0.25886524822695034, - "acc_stderr": 0.026129572527180848, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.23404255319148937, + "acc_stderr": 0.025257861359432397, + "brier_score": 0.7687279343923068, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_college_physics": { - "acc": 0.19607843137254902, - "acc_stderr": 0.039505818611799616, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.2549019607843137, + "acc_stderr": 0.04336432707993176, + "brier_score": 0.7717036537544136, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_high_school_physics": { - "acc": 0.304635761589404, - "acc_stderr": 0.03757949922943343, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.2847682119205298, + "acc_stderr": 0.03684881521389023, + "brier_score": 0.7465195131792135, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_high_school_biology": { - "acc": 0.33225806451612905, - "acc_stderr": 0.026795560848122787, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.2645161290322581, + "acc_stderr": 0.02509189237885928, + "brier_score": 0.7642024621618707, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_college_biology": { - "acc": 0.2916666666666667, - "acc_stderr": 0.03800968060554858, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.2361111111111111, + "acc_stderr": 0.03551446610810826, + "brier_score": 0.7616838225475571, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_anatomy": { - "acc": 0.2518518518518518, - "acc_stderr": 0.03749850709174021, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.28888888888888886, + "acc_stderr": 0.0391545063041425, + "brier_score": 0.7611247992529132, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_college_chemistry": { - "acc": 0.27, - "acc_stderr": 0.044619604333847394, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.23, + "acc_stderr": 0.04229525846816506, + "brier_score": 0.7606467477392989, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_computer_security": { - "acc": 0.28, - "acc_stderr": 0.04512608598542127, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.32, + "acc_stderr": 0.04688261722621504, + "brier_score": 0.7515781654981363, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_college_computer_science": { - "acc": 0.31, - "acc_stderr": 0.04648231987117316, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.26, + "acc_stderr": 0.04408440022768077, + "brier_score": 0.7662889507808877, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_astronomy": { - "acc": 0.21710526315789475, - "acc_stderr": 0.03355045304882924, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.2894736842105263, + "acc_stderr": 0.036906779861372814, + "brier_score": 0.774228413944404, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_college_mathematics": { - "acc": 0.25, - "acc_stderr": 0.04351941398892446, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.19, + "acc_stderr": 0.03942772444036623, + "brier_score": 0.7655113831829723, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_conceptual_physics": { - "acc": 0.28085106382978725, - "acc_stderr": 0.02937917046412482, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.28936170212765955, + "acc_stderr": 0.02964400657700962, + "brier_score": 0.7505745859976238, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_abstract_algebra": { - "acc": 0.25, - "acc_stderr": 0.04351941398892446, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "brier_score": 0.7628138033789061, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_high_school_computer_science": { - "acc": 0.29, - "acc_stderr": 0.04560480215720683, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.26, + "acc_stderr": 0.044084400227680794, + "brier_score": 0.7548514925906404, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_machine_learning": { - "acc": 0.23214285714285715, - "acc_stderr": 0.04007341809755806, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.20535714285714285, + "acc_stderr": 0.03834241021419072, + "brier_score": 0.7589028909280103, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_high_school_chemistry": { - "acc": 0.30049261083743845, - "acc_stderr": 0.03225799476233485, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.21674876847290642, + "acc_stderr": 0.028990331252516235, + "brier_score": 0.7688436179237746, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_high_school_statistics": { - "acc": 0.4212962962962963, - "acc_stderr": 0.03367462138896078, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.2824074074074074, + "acc_stderr": 0.030701372111510934, + "brier_score": 0.7584148541273571, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_elementary_mathematics": { - "acc": 0.2671957671957672, - "acc_stderr": 0.02278967314577657, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.24074074074074073, + "acc_stderr": 0.02201908001221791, + "brier_score": 0.7639398310366289, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_electrical_engineering": { - "acc": 0.21379310344827587, - "acc_stderr": 0.034165204477475494, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.2482758620689655, + "acc_stderr": 0.036001056927277696, + "brier_score": 0.7610945759264104, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "mmlu_high_school_mathematics": { - "acc": 0.25555555555555554, - "acc_stderr": 0.02659393910184407, - "timestamp": "2024-11-24T16-59-05.111235" + "acc": 0.2814814814814815, + "acc_stderr": 0.02742001935094527, + "brier_score": 0.76561675444009, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-20-50.109941" }, "arc_challenge": { "acc": 0.310580204778157, diff --git a/metadata/model_scores/results_google__gemma-2-2b.json b/metadata/model_scores/results_google__gemma-2-2b.json index 2a92165..2f96ddc 100644 --- a/metadata/model_scores/results_google__gemma-2-2b.json +++ b/metadata/model_scores/results_google__gemma-2-2b.json @@ -1,292 +1,406 @@ { "model_name": "google__gemma-2-2b", - "last_updated": "2024-12-04 11:25:34.665720", + "last_updated": "2024-12-19 13:41:46.459626", "results": { "harness": { "mmlu_world_religions": { - "acc": 0.695906432748538, - "acc_stderr": 0.03528211258245231, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.631578947368421, + "acc_stderr": 0.036996580176568775, + "brier_score": 0.46319946309617904, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_formal_logic": { - "acc": 0.3253968253968254, - "acc_stderr": 0.041905964388711366, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.373015873015873, + "acc_stderr": 0.04325506042017087, + "brier_score": 0.7294773962828401, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_prehistory": { - "acc": 0.5679012345679012, - "acc_stderr": 0.027563010971606676, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.5833333333333334, + "acc_stderr": 0.027431623722415, + "brier_score": 0.5552776145606315, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_moral_scenarios": { - "acc": 0.2759776536312849, - "acc_stderr": 0.014950103002475358, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.2424581005586592, + "acc_stderr": 0.014333522059217887, + "brier_score": 0.7575592851431834, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_high_school_world_history": { - "acc": 0.6666666666666666, - "acc_stderr": 0.0306858205966108, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.6075949367088608, + "acc_stderr": 0.03178471874564729, + "brier_score": 0.5543665383004351, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_moral_disputes": { - "acc": 0.6040462427745664, - "acc_stderr": 0.02632981334194624, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.5346820809248555, + "acc_stderr": 0.026854257928258872, + "brier_score": 0.6043277638860677, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_professional_law": { - "acc": 0.4067796610169492, - "acc_stderr": 0.01254632559656952, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.3774445893089961, + "acc_stderr": 0.01238068091116581, + "brier_score": 0.7120462823680153, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_logical_fallacies": { - "acc": 0.6073619631901841, - "acc_stderr": 0.03836740907831029, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.5398773006134969, + "acc_stderr": 0.0391585729143697, + "brier_score": 0.5600561413857739, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_high_school_us_history": { - "acc": 0.6715686274509803, - "acc_stderr": 0.03296245110172227, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.5980392156862745, + "acc_stderr": 0.034411900234824655, + "brier_score": 0.5250723628644467, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_philosophy": { - "acc": 0.5852090032154341, - "acc_stderr": 0.02798268045975956, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.5594855305466238, + "acc_stderr": 0.02819640057419743, + "brier_score": 0.57333295776366, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_jurisprudence": { - "acc": 0.6018518518518519, - "acc_stderr": 0.04732332615978815, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.5555555555555556, + "acc_stderr": 0.04803752235190193, + "brier_score": 0.5539311191634335, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_international_law": { - "acc": 0.6859504132231405, - "acc_stderr": 0.042369647530410184, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.6198347107438017, + "acc_stderr": 0.04431324501968432, + "brier_score": 0.517865160418074, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_high_school_european_history": { - "acc": 0.6666666666666666, - "acc_stderr": 0.036810508691615486, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.5878787878787879, + "acc_stderr": 0.03843566993588717, + "brier_score": 0.5555570317544237, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_high_school_government_and_politics": { - "acc": 0.772020725388601, - "acc_stderr": 0.030276909945178263, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.6787564766839378, + "acc_stderr": 0.033699508685490674, + "brier_score": 0.4470321087645973, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_high_school_microeconomics": { - "acc": 0.6134453781512605, - "acc_stderr": 0.031631458075523776, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.49159663865546216, + "acc_stderr": 0.03247390276569669, + "brier_score": 0.583716264421914, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_high_school_geography": { - "acc": 0.7474747474747475, - "acc_stderr": 0.030954055470365907, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.601010101010101, + "acc_stderr": 0.0348890161685273, + "brier_score": 0.4966484789683147, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_high_school_psychology": { - "acc": 0.7394495412844037, - "acc_stderr": 0.018819182034850068, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.708256880733945, + "acc_stderr": 0.019489300968876532, + "brier_score": 0.4210221460891557, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_public_relations": { - "acc": 0.6, - "acc_stderr": 0.0469237132203465, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.5454545454545454, + "acc_stderr": 0.04769300568972744, + "brier_score": 0.5557865683408226, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_us_foreign_policy": { - "acc": 0.72, - "acc_stderr": 0.04512608598542128, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.74, + "acc_stderr": 0.0440844002276808, + "brier_score": 0.3961873807762082, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_sociology": { - "acc": 0.7512437810945274, - "acc_stderr": 0.030567675938916718, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.7114427860696517, + "acc_stderr": 0.03203841040213321, + "brier_score": 0.4039414809868519, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_high_school_macroeconomics": { - "acc": 0.5307692307692308, - "acc_stderr": 0.025302958890850154, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.47435897435897434, + "acc_stderr": 0.025317649726448652, + "brier_score": 0.6295952852373625, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_security_studies": { "acc": 0.5918367346938775, "acc_stderr": 0.03146465712827424, - "timestamp": "2024-11-21T18-24-59.376782" + "brier_score": 0.5188297835626492, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_professional_psychology": { - "acc": 0.5441176470588235, - "acc_stderr": 0.020148939420415752, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.4803921568627451, + "acc_stderr": 0.020212274976302957, + "brier_score": 0.6453178177882781, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_human_sexuality": { - "acc": 0.5954198473282443, - "acc_stderr": 0.043046937953806645, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.5877862595419847, + "acc_stderr": 0.043171711948702556, + "brier_score": 0.5239511420600674, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_econometrics": { - "acc": 0.32456140350877194, - "acc_stderr": 0.04404556157374767, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.3157894736842105, + "acc_stderr": 0.04372748290278007, + "brier_score": 0.7487366917955591, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_miscellaneous": { - "acc": 0.6704980842911877, - "acc_stderr": 0.01680832226174047, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.6283524904214559, + "acc_stderr": 0.017280802522133182, + "brier_score": 0.46974857615740373, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_marketing": { - "acc": 0.7948717948717948, - "acc_stderr": 0.026453508054040346, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.7692307692307693, + "acc_stderr": 0.027601921381417573, + "brier_score": 0.33759788300342536, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_management": { - "acc": 0.6893203883495146, - "acc_stderr": 0.0458212416016155, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.6601941747572816, + "acc_stderr": 0.046897659372781335, + "brier_score": 0.45912572812495833, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_nutrition": { - "acc": 0.6045751633986928, - "acc_stderr": 0.02799672318063145, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.5718954248366013, + "acc_stderr": 0.028332397483664264, + "brier_score": 0.5576880805023428, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_medical_genetics": { - "acc": 0.7, - "acc_stderr": 0.046056618647183814, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.62, + "acc_stderr": 0.04878317312145632, + "brier_score": 0.5127618333715876, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_human_aging": { - "acc": 0.5874439461883408, - "acc_stderr": 0.03304062175449297, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.5964125560538116, + "acc_stderr": 0.03292802819330314, + "brier_score": 0.5485953527985196, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_professional_medicine": { - "acc": 0.41544117647058826, - "acc_stderr": 0.029935342707877746, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.4889705882352941, + "acc_stderr": 0.030365446477275675, + "brier_score": 0.6471881105163275, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_college_medicine": { - "acc": 0.6127167630057804, - "acc_stderr": 0.03714325906302065, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.5549132947976878, + "acc_stderr": 0.03789401760283647, + "brier_score": 0.5791128685234473, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_business_ethics": { - "acc": 0.54, - "acc_stderr": 0.05009082659620332, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.48, + "acc_stderr": 0.050211673156867795, + "brier_score": 0.6154668985699773, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_clinical_knowledge": { - "acc": 0.5811320754716981, - "acc_stderr": 0.0303650508291152, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.5660377358490566, + "acc_stderr": 0.030503292013342596, + "brier_score": 0.540663045764408, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_global_facts": { - "acc": 0.3, - "acc_stderr": 0.046056618647183814, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "brier_score": 0.7444249433611936, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_virology": { - "acc": 0.46987951807228917, - "acc_stderr": 0.03885425420866766, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.4879518072289157, + "acc_stderr": 0.03891364495835819, + "brier_score": 0.683722495071859, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_professional_accounting": { - "acc": 0.3971631205673759, - "acc_stderr": 0.0291898056735871, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.38652482269503546, + "acc_stderr": 0.029049190342543458, + "brier_score": 0.699502653053553, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_college_physics": { - "acc": 0.38235294117647056, - "acc_stderr": 0.04835503696107223, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.4019607843137255, + "acc_stderr": 0.048786087144669976, + "brier_score": 0.6956491833879586, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_high_school_physics": { - "acc": 0.40397350993377484, - "acc_stderr": 0.04006485685365342, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.37748344370860926, + "acc_stderr": 0.039580272311215706, + "brier_score": 0.7227633978689442, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_high_school_biology": { - "acc": 0.6709677419354839, - "acc_stderr": 0.026729499068349954, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.6387096774193548, + "acc_stderr": 0.02732754844795753, + "brier_score": 0.4647084689850728, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_college_biology": { - "acc": 0.6180555555555556, - "acc_stderr": 0.040629907841466674, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.5763888888888888, + "acc_stderr": 0.0413212501972337, + "brier_score": 0.5124225303466369, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_anatomy": { - "acc": 0.5185185185185185, - "acc_stderr": 0.043163785995113245, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.5481481481481482, + "acc_stderr": 0.04299268905480864, + "brier_score": 0.5658901320145272, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_college_chemistry": { - "acc": 0.42, - "acc_stderr": 0.04960449637488583, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.43, + "acc_stderr": 0.049756985195624284, + "brier_score": 0.6452781056531832, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_computer_security": { - "acc": 0.64, - "acc_stderr": 0.048241815132442176, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.57, + "acc_stderr": 0.049756985195624284, + "brier_score": 0.5812758953100281, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_college_computer_science": { - "acc": 0.45, - "acc_stderr": 0.05, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.46, + "acc_stderr": 0.05009082659620333, + "brier_score": 0.6781639353860459, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_astronomy": { - "acc": 0.5657894736842105, - "acc_stderr": 0.0403356566784832, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.5131578947368421, + "acc_stderr": 0.04067533136309172, + "brier_score": 0.5734478143657858, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_college_mathematics": { "acc": 0.32, - "acc_stderr": 0.04688261722621504, - "timestamp": "2024-11-21T18-24-59.376782" + "acc_stderr": 0.046882617226215034, + "brier_score": 0.746728079111178, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_conceptual_physics": { - "acc": 0.4723404255319149, - "acc_stderr": 0.03263597118409769, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.425531914893617, + "acc_stderr": 0.03232146916224469, + "brier_score": 0.6784287911825118, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_abstract_algebra": { - "acc": 0.26, - "acc_stderr": 0.04408440022768078, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.29, + "acc_stderr": 0.045604802157206845, + "brier_score": 0.7464991036605416, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_high_school_computer_science": { - "acc": 0.48, - "acc_stderr": 0.050211673156867795, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.39, + "acc_stderr": 0.04902071300001975, + "brier_score": 0.6501496583396172, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_machine_learning": { - "acc": 0.3392857142857143, - "acc_stderr": 0.04493949068613539, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.3125, + "acc_stderr": 0.043994650575715215, + "brier_score": 0.7544617910901751, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_high_school_chemistry": { - "acc": 0.4729064039408867, - "acc_stderr": 0.03512819077876106, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.39901477832512317, + "acc_stderr": 0.03445487686264716, + "brier_score": 0.6645080178436065, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_high_school_statistics": { - "acc": 0.46296296296296297, - "acc_stderr": 0.03400603625538271, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.4166666666666667, + "acc_stderr": 0.033622774366080424, + "brier_score": 0.6974244532546542, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_elementary_mathematics": { - "acc": 0.3439153439153439, - "acc_stderr": 0.024464426625596426, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.30952380952380953, + "acc_stderr": 0.023809523809523846, + "brier_score": 0.7455024514630606, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_electrical_engineering": { - "acc": 0.6068965517241379, - "acc_stderr": 0.0407032901370707, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.5517241379310345, + "acc_stderr": 0.04144311810878152, + "brier_score": 0.5825828767130868, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "mmlu_high_school_mathematics": { - "acc": 0.3, - "acc_stderr": 0.02794045713622842, - "timestamp": "2024-11-21T18-24-59.376782" + "acc": 0.27037037037037037, + "acc_stderr": 0.027080372815145658, + "brier_score": 0.7599156637458062, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-31-51.974891" }, "arc_challenge": { "acc": 0.5008532423208191, diff --git a/metadata/model_scores/results_google__gemma-7b.json b/metadata/model_scores/results_google__gemma-7b.json index 1c62c01..99414f2 100644 --- a/metadata/model_scores/results_google__gemma-7b.json +++ b/metadata/model_scores/results_google__gemma-7b.json @@ -1,292 +1,406 @@ { "model_name": "google__gemma-7b", - "last_updated": "2024-12-04 11:25:15.123019", + "last_updated": "2024-12-19 13:41:20.232884", "results": { "harness": { "mmlu_world_religions": { - "acc": 0.8421052631578947, - "acc_stderr": 0.027966785859160865, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.8362573099415205, + "acc_stderr": 0.028380919596145866, + "brier_score": 0.23192067938649402, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_formal_logic": { - "acc": 0.5, - "acc_stderr": 0.04472135954999579, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.4523809523809524, + "acc_stderr": 0.044518079590553275, + "brier_score": 0.6492991823803103, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_prehistory": { - "acc": 0.7314814814814815, - "acc_stderr": 0.024659685185967273, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.7067901234567902, + "acc_stderr": 0.02532988817190092, + "brier_score": 0.36084739252752485, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_moral_scenarios": { - "acc": 0.31508379888268156, - "acc_stderr": 0.015536850852473635, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.24916201117318434, + "acc_stderr": 0.014465893829859933, + "brier_score": 0.8291813080992225, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_high_school_world_history": { "acc": 0.8227848101265823, - "acc_stderr": 0.024856364184503228, - "timestamp": "2024-11-22T16-58-09.137194" + "acc_stderr": 0.024856364184503234, + "brier_score": 0.26232318467685783, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_moral_disputes": { - "acc": 0.6676300578034682, - "acc_stderr": 0.02536116874968822, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.653179190751445, + "acc_stderr": 0.02562472399403046, + "brier_score": 0.4428304040800636, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_professional_law": { - "acc": 0.4661016949152542, - "acc_stderr": 0.012740853872949832, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.470013037809648, + "acc_stderr": 0.01274724896707906, + "brier_score": 0.6545306376603126, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_logical_fallacies": { "acc": 0.7484662576687117, "acc_stderr": 0.03408997886857529, - "timestamp": "2024-11-22T16-58-09.137194" + "brier_score": 0.3275554200833385, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_high_school_us_history": { - "acc": 0.7843137254901961, - "acc_stderr": 0.028867431449849313, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.7745098039215687, + "acc_stderr": 0.029331162294251735, + "brier_score": 0.2932653335661421, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_philosophy": { - "acc": 0.7009646302250804, - "acc_stderr": 0.026003301117885128, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.7363344051446945, + "acc_stderr": 0.02502553850053234, + "brier_score": 0.37893514894575386, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_jurisprudence": { - "acc": 0.75, - "acc_stderr": 0.04186091791394607, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.7222222222222222, + "acc_stderr": 0.043300437496507416, + "brier_score": 0.37148916048633085, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_international_law": { - "acc": 0.8512396694214877, - "acc_stderr": 0.03248470083807195, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.7933884297520661, + "acc_stderr": 0.036959801280988254, + "brier_score": 0.30941704283786436, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_high_school_european_history": { - "acc": 0.7575757575757576, - "acc_stderr": 0.03346409881055953, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.7757575757575758, + "acc_stderr": 0.03256866661681102, + "brier_score": 0.3419807011294865, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_high_school_government_and_politics": { - "acc": 0.8860103626943006, - "acc_stderr": 0.022935144053919436, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.844559585492228, + "acc_stderr": 0.026148483469153324, + "brier_score": 0.20123268843700662, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_high_school_microeconomics": { - "acc": 0.6470588235294118, - "acc_stderr": 0.031041941304059278, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.6302521008403361, + "acc_stderr": 0.031357095996135904, + "brier_score": 0.5082437444997995, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_high_school_geography": { - "acc": 0.803030303030303, - "acc_stderr": 0.028335609732463362, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.8080808080808081, + "acc_stderr": 0.028057791672989017, + "brier_score": 0.26562395906533987, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_high_school_psychology": { "acc": 0.8128440366972477, - "acc_stderr": 0.01672268452620016, - "timestamp": "2024-11-22T16-58-09.137194" + "acc_stderr": 0.016722684526200144, + "brier_score": 0.26215102561607717, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_public_relations": { - "acc": 0.6545454545454545, - "acc_stderr": 0.04554619617541054, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.6363636363636364, + "acc_stderr": 0.04607582090719976, + "brier_score": 0.4314491872734749, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_us_foreign_policy": { "acc": 0.87, - "acc_stderr": 0.03379976689896308, - "timestamp": "2024-11-22T16-58-09.137194" + "acc_stderr": 0.03379976689896309, + "brier_score": 0.1993096348099571, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_sociology": { - "acc": 0.835820895522388, - "acc_stderr": 0.026193923544454125, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.7910447761194029, + "acc_stderr": 0.028748298931728658, + "brier_score": 0.26865300694882494, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_high_school_macroeconomics": { - "acc": 0.6307692307692307, - "acc_stderr": 0.02446861524147891, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.6256410256410256, + "acc_stderr": 0.024537591572830517, + "brier_score": 0.4942070941658129, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_security_studies": { - "acc": 0.7183673469387755, - "acc_stderr": 0.028795185574291296, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.7387755102040816, + "acc_stderr": 0.028123429335142797, + "brier_score": 0.3675529070865265, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_professional_psychology": { - "acc": 0.6895424836601307, - "acc_stderr": 0.018718067052623213, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.6601307189542484, + "acc_stderr": 0.019162418588623553, + "brier_score": 0.4584601558942431, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_human_sexuality": { - "acc": 0.7251908396946565, - "acc_stderr": 0.03915345408847836, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.816793893129771, + "acc_stderr": 0.033927709264947335, + "brier_score": 0.33801186706674946, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_econometrics": { - "acc": 0.45614035087719296, - "acc_stderr": 0.04685473041907789, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.47368421052631576, + "acc_stderr": 0.046970851366478626, + "brier_score": 0.6289216779161965, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_miscellaneous": { - "acc": 0.8416347381864623, - "acc_stderr": 0.013055346753516743, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.8148148148148148, + "acc_stderr": 0.013890862162876164, + "brier_score": 0.254727901877134, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_marketing": { - "acc": 0.8931623931623932, - "acc_stderr": 0.020237149008990943, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.8675213675213675, + "acc_stderr": 0.022209309073165626, + "brier_score": 0.1880570882030185, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_management": { - "acc": 0.8252427184466019, - "acc_stderr": 0.03760178006026621, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.8446601941747572, + "acc_stderr": 0.03586594738573975, + "brier_score": 0.21054564782275975, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_nutrition": { - "acc": 0.7418300653594772, - "acc_stderr": 0.025058503316958147, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.6993464052287581, + "acc_stderr": 0.026256053835718964, + "brier_score": 0.3912421136861796, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_medical_genetics": { - "acc": 0.75, - "acc_stderr": 0.04351941398892446, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.72, + "acc_stderr": 0.045126085985421276, + "brier_score": 0.3520073746839315, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_human_aging": { - "acc": 0.7085201793721974, - "acc_stderr": 0.03050028317654584, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.6905829596412556, + "acc_stderr": 0.031024411740572203, + "brier_score": 0.41113365407849767, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_professional_medicine": { - "acc": 0.5772058823529411, - "acc_stderr": 0.03000856284500347, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.6102941176470589, + "acc_stderr": 0.02962466358115969, + "brier_score": 0.49532887960076133, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_college_medicine": { - "acc": 0.6473988439306358, - "acc_stderr": 0.03643037168958548, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.6069364161849711, + "acc_stderr": 0.03724249595817731, + "brier_score": 0.4769229917453633, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_business_ethics": { - "acc": 0.64, - "acc_stderr": 0.048241815132442176, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.65, + "acc_stderr": 0.0479372485441102, + "brier_score": 0.45469680318876826, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_clinical_knowledge": { - "acc": 0.660377358490566, - "acc_stderr": 0.029146904747798328, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.7132075471698113, + "acc_stderr": 0.027834912527544074, + "brier_score": 0.398876507521786, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_global_facts": { - "acc": 0.35, - "acc_stderr": 0.0479372485441102, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.36, + "acc_stderr": 0.048241815132442176, + "brier_score": 0.693773285966367, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_virology": { - "acc": 0.5240963855421686, - "acc_stderr": 0.03887971849597264, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.5180722891566265, + "acc_stderr": 0.038899512528272166, + "brier_score": 0.6993771685259159, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_professional_accounting": { - "acc": 0.46099290780141844, - "acc_stderr": 0.02973659252642444, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.5212765957446809, + "acc_stderr": 0.029800481645628693, + "brier_score": 0.5919019586393015, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_college_physics": { - "acc": 0.38235294117647056, - "acc_stderr": 0.04835503696107223, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.3431372549019608, + "acc_stderr": 0.04724007352383889, + "brier_score": 0.666476923757466, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_high_school_physics": { - "acc": 0.3973509933774834, - "acc_stderr": 0.03995524007681681, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.36423841059602646, + "acc_stderr": 0.03929111781242742, + "brier_score": 0.6936531576790718, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_high_school_biology": { - "acc": 0.8, - "acc_stderr": 0.02275520495954294, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.7612903225806451, + "acc_stderr": 0.024251071262208837, + "brier_score": 0.3246457688347233, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_college_biology": { - "acc": 0.7291666666666666, - "acc_stderr": 0.03716177437566017, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.7708333333333334, + "acc_stderr": 0.035146974678623884, + "brier_score": 0.3102445873248153, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_anatomy": { - "acc": 0.5259259259259259, - "acc_stderr": 0.04313531696750575, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.5703703703703704, + "acc_stderr": 0.04276349494376599, + "brier_score": 0.5425841934637702, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_college_chemistry": { - "acc": 0.48, - "acc_stderr": 0.050211673156867795, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.45, + "acc_stderr": 0.04999999999999999, + "brier_score": 0.6260057241013216, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_computer_security": { - "acc": 0.75, - "acc_stderr": 0.04351941398892446, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.69, + "acc_stderr": 0.04648231987117316, + "brier_score": 0.41333249631795255, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_college_computer_science": { - "acc": 0.49, - "acc_stderr": 0.050241839379569095, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.55, + "acc_stderr": 0.049999999999999996, + "brier_score": 0.5801512561734236, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_astronomy": { - "acc": 0.6907894736842105, - "acc_stderr": 0.03761070869867479, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.7302631578947368, + "acc_stderr": 0.03611780560284898, + "brier_score": 0.37540527560539916, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_college_mathematics": { - "acc": 0.38, - "acc_stderr": 0.04878317312145632, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.39, + "acc_stderr": 0.04902071300001974, + "brier_score": 0.7107035113276393, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_conceptual_physics": { - "acc": 0.5829787234042553, - "acc_stderr": 0.03223276266711712, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.574468085106383, + "acc_stderr": 0.03232146916224469, + "brier_score": 0.5277388052784279, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_abstract_algebra": { "acc": 0.24, - "acc_stderr": 0.04292346959909282, - "timestamp": "2024-11-22T16-58-09.137194" + "acc_stderr": 0.042923469599092816, + "brier_score": 0.7316091176070936, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_high_school_computer_science": { - "acc": 0.65, - "acc_stderr": 0.047937248544110196, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.68, + "acc_stderr": 0.04688261722621504, + "brier_score": 0.4363723609398192, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_machine_learning": { - "acc": 0.5535714285714286, - "acc_stderr": 0.047184714852195865, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.49107142857142855, + "acc_stderr": 0.04745033255489123, + "brier_score": 0.6216939745012614, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_high_school_chemistry": { - "acc": 0.5024630541871922, - "acc_stderr": 0.03517945038691063, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.5221674876847291, + "acc_stderr": 0.035145285621750094, + "brier_score": 0.560523454378963, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_high_school_statistics": { - "acc": 0.48148148148148145, - "acc_stderr": 0.034076320938540516, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.5462962962962963, + "acc_stderr": 0.03395322726375797, + "brier_score": 0.5618044455674843, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_elementary_mathematics": { - "acc": 0.4603174603174603, - "acc_stderr": 0.025670080636909186, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.4312169312169312, + "acc_stderr": 0.025506481698138215, + "brier_score": 0.6506498011362429, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_electrical_engineering": { - "acc": 0.6344827586206897, - "acc_stderr": 0.04013124195424386, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.5448275862068965, + "acc_stderr": 0.04149886942192117, + "brier_score": 0.5214244847001162, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "mmlu_high_school_mathematics": { - "acc": 0.362962962962963, - "acc_stderr": 0.02931820364520686, - "timestamp": "2024-11-22T16-58-09.137194" + "acc": 0.35185185185185186, + "acc_stderr": 0.029116617606083018, + "brier_score": 0.6953923313703757, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-19-36.388618" }, "arc_challenge": { "acc": 0.5708191126279863, diff --git a/metadata/model_scores/results_jb723__LLaMA2-en-ko-7B-model.json b/metadata/model_scores/results_jb723__LLaMA2-en-ko-7B-model.json index 00aeb55..ec0bdce 100644 --- a/metadata/model_scores/results_jb723__LLaMA2-en-ko-7B-model.json +++ b/metadata/model_scores/results_jb723__LLaMA2-en-ko-7B-model.json @@ -1,6 +1,6 @@ { "model_name": "jb723/LLaMA2-en-ko-7B-model", - "last_updated": "2024-12-04 11:23:38.857697", + "last_updated": "2024-12-19 13:39:03.610739", "results": { "harness": { "minerva_math_precalc": { @@ -228,289 +228,403 @@ "timestamp": "2024-06-06T19-51-27.840893" }, "mmlu_world_religions": { - "acc": 0.672514619883041, - "acc_stderr": 0.035993357714560276, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.6783625730994152, + "acc_stderr": 0.03582529442573122, + "brier_score": 0.534748864523759, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_formal_logic": { - "acc": 0.24603174603174602, - "acc_stderr": 0.038522733649243156, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.3253968253968254, + "acc_stderr": 0.041905964388711366, + "brier_score": 0.8666146302504888, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_prehistory": { - "acc": 0.5154320987654321, - "acc_stderr": 0.027807490044276184, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.48148148148148145, + "acc_stderr": 0.027801656212323674, + "brier_score": 0.7871000666395396, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_moral_scenarios": { "acc": 0.23798882681564246, "acc_stderr": 0.014242630070574885, - "timestamp": "2024-11-15T06-50-09.288337" + "brier_score": 0.8958739388927954, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_high_school_world_history": { - "acc": 0.6751054852320675, - "acc_stderr": 0.030486039389105307, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.5907172995780591, + "acc_stderr": 0.032007041833595914, + "brier_score": 0.5286244391845626, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_moral_disputes": { - "acc": 0.5260115606936416, - "acc_stderr": 0.02688264343402289, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.45664739884393063, + "acc_stderr": 0.02681771813034892, + "brier_score": 0.7386694527340083, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_professional_law": { - "acc": 0.3546284224250326, - "acc_stderr": 0.012218576439090174, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.35071707953063885, + "acc_stderr": 0.012187773370741518, + "brier_score": 0.8635056976637175, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_logical_fallacies": { - "acc": 0.4785276073619632, - "acc_stderr": 0.0392474687675113, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.50920245398773, + "acc_stderr": 0.03927705600787443, + "brier_score": 0.6678415022899983, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_high_school_us_history": { - "acc": 0.5637254901960784, - "acc_stderr": 0.03480693138457038, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.553921568627451, + "acc_stderr": 0.03488845451304974, + "brier_score": 0.5595603036672312, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_philosophy": { - "acc": 0.5594855305466238, - "acc_stderr": 0.02819640057419743, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.5176848874598071, + "acc_stderr": 0.028380322849077138, + "brier_score": 0.6871936008769992, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_jurisprudence": { - "acc": 0.49074074074074076, - "acc_stderr": 0.04832853553437055, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.5185185185185185, + "acc_stderr": 0.04830366024635331, + "brier_score": 0.6878288469289898, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_international_law": { - "acc": 0.6033057851239669, - "acc_stderr": 0.04465869780531009, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.5950413223140496, + "acc_stderr": 0.04481137755942469, + "brier_score": 0.5623984217998076, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_high_school_european_history": { - "acc": 0.5757575757575758, - "acc_stderr": 0.038592681420702636, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.5636363636363636, + "acc_stderr": 0.03872592983524754, + "brier_score": 0.6111700359415961, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_high_school_government_and_politics": { - "acc": 0.6787564766839378, - "acc_stderr": 0.033699508685490674, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.6217616580310881, + "acc_stderr": 0.03499807276193338, + "brier_score": 0.5171590492344016, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_high_school_microeconomics": { - "acc": 0.41596638655462187, - "acc_stderr": 0.03201650100739615, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.3949579831932773, + "acc_stderr": 0.03175367846096625, + "brier_score": 0.8008431040907894, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_high_school_geography": { - "acc": 0.5909090909090909, - "acc_stderr": 0.03502975799413007, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.5, + "acc_stderr": 0.035623524993954825, + "brier_score": 0.716531768993534, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_high_school_psychology": { - "acc": 0.6201834862385321, - "acc_stderr": 0.020808825617866244, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.6055045871559633, + "acc_stderr": 0.020954642108587485, + "brier_score": 0.5735431778963822, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_public_relations": { - "acc": 0.5545454545454546, - "acc_stderr": 0.047605488214603246, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.4727272727272727, + "acc_stderr": 0.04782001791380063, + "brier_score": 0.7798231001117422, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_us_foreign_policy": { - "acc": 0.64, - "acc_stderr": 0.04824181513244218, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.63, + "acc_stderr": 0.04852365870939099, + "brier_score": 0.5615693250727137, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_sociology": { - "acc": 0.5870646766169154, - "acc_stderr": 0.03481520803367348, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.7164179104477612, + "acc_stderr": 0.03187187537919798, + "brier_score": 0.46012516626124617, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_high_school_macroeconomics": { "acc": 0.44358974358974357, - "acc_stderr": 0.025189149894764194, - "timestamp": "2024-11-15T06-50-09.288337" + "acc_stderr": 0.0251891498947642, + "brier_score": 0.7532580528185948, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_security_studies": { - "acc": 0.4489795918367347, + "acc": 0.5510204081632653, "acc_stderr": 0.03184213866687579, - "timestamp": "2024-11-15T06-50-09.288337" + "brier_score": 0.6568185057962493, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_professional_psychology": { - "acc": 0.4477124183006536, - "acc_stderr": 0.020116925347422425, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.43790849673202614, + "acc_stderr": 0.020071257886886525, + "brier_score": 0.7805813506963591, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_human_sexuality": { - "acc": 0.549618320610687, - "acc_stderr": 0.04363643698524779, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.5801526717557252, + "acc_stderr": 0.04328577215262972, + "brier_score": 0.6283695295776315, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_econometrics": { - "acc": 0.32456140350877194, - "acc_stderr": 0.04404556157374767, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.23684210526315788, + "acc_stderr": 0.03999423879281336, + "brier_score": 1.040054312603874, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_miscellaneous": { - "acc": 0.6360153256704981, - "acc_stderr": 0.017205684809032232, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.632183908045977, + "acc_stderr": 0.017243828891846273, + "brier_score": 0.5591275539615225, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_marketing": { - "acc": 0.6623931623931624, - "acc_stderr": 0.030980296992618558, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.6709401709401709, + "acc_stderr": 0.03078232157768818, + "brier_score": 0.5298966348501603, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_management": { - "acc": 0.5825242718446602, - "acc_stderr": 0.048828405482122375, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.6116504854368932, + "acc_stderr": 0.04825729337356391, + "brier_score": 0.6290326612663383, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_nutrition": { - "acc": 0.4803921568627451, - "acc_stderr": 0.028607893699576063, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.4738562091503268, + "acc_stderr": 0.028590752958852394, + "brier_score": 0.7600916780576064, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_medical_genetics": { - "acc": 0.56, - "acc_stderr": 0.049888765156985884, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.47, + "acc_stderr": 0.05016135580465919, + "brier_score": 0.7919592716712658, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_human_aging": { - "acc": 0.5336322869955157, - "acc_stderr": 0.03348180017060306, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.4170403587443946, + "acc_stderr": 0.03309266936071721, + "brier_score": 0.764915455584205, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_professional_medicine": { - "acc": 0.4338235294117647, - "acc_stderr": 0.030105636570016636, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.4227941176470588, + "acc_stderr": 0.030008562845003476, + "brier_score": 0.7693848977296722, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_college_medicine": { "acc": 0.43352601156069365, - "acc_stderr": 0.03778621079092056, - "timestamp": "2024-11-15T06-50-09.288337" + "acc_stderr": 0.037786210790920545, + "brier_score": 0.7900118197708276, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_business_ethics": { - "acc": 0.48, + "acc": 0.52, "acc_stderr": 0.050211673156867795, - "timestamp": "2024-11-15T06-50-09.288337" + "brier_score": 0.690535414285433, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_clinical_knowledge": { - "acc": 0.4679245283018868, - "acc_stderr": 0.030709486992556552, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.4830188679245283, + "acc_stderr": 0.030755120364119905, + "brier_score": 0.7388561021827996, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_global_facts": { - "acc": 0.39, - "acc_stderr": 0.04902071300001974, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.32, + "acc_stderr": 0.046882617226215034, + "brier_score": 0.8966340480041656, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_virology": { - "acc": 0.4397590361445783, - "acc_stderr": 0.03864139923699122, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.4457831325301205, + "acc_stderr": 0.03869543323472101, + "brier_score": 0.889841462632872, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_professional_accounting": { - "acc": 0.3617021276595745, - "acc_stderr": 0.028663820147199492, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.36879432624113473, + "acc_stderr": 0.028782227561347257, + "brier_score": 0.821098937892523, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_college_physics": { - "acc": 0.14705882352941177, - "acc_stderr": 0.035240689515674495, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.27450980392156865, + "acc_stderr": 0.044405219061793275, + "brier_score": 0.9858278163501498, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_high_school_physics": { - "acc": 0.31788079470198677, - "acc_stderr": 0.038020397601079024, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.23841059602649006, + "acc_stderr": 0.03479185572599661, + "brier_score": 0.9301743583498789, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_high_school_biology": { - "acc": 0.5, - "acc_stderr": 0.028444006199428714, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.4870967741935484, + "acc_stderr": 0.02843453315268186, + "brier_score": 0.7268153093495426, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_college_biology": { - "acc": 0.4652777777777778, - "acc_stderr": 0.04171115858181618, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.4166666666666667, + "acc_stderr": 0.041227287076512825, + "brier_score": 0.807682603750939, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_anatomy": { - "acc": 0.4444444444444444, - "acc_stderr": 0.04292596718256981, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.5111111111111111, + "acc_stderr": 0.04318275491977976, + "brier_score": 0.7261012587186336, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_college_chemistry": { - "acc": 0.37, - "acc_stderr": 0.048523658709391, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.36, + "acc_stderr": 0.04824181513244218, + "brier_score": 0.7860073437698719, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_computer_security": { "acc": 0.51, "acc_stderr": 0.05024183937956911, - "timestamp": "2024-11-15T06-50-09.288337" + "brier_score": 0.6798467302926896, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_college_computer_science": { - "acc": 0.35, - "acc_stderr": 0.047937248544110196, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "brier_score": 0.858095517218403, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_astronomy": { - "acc": 0.4407894736842105, - "acc_stderr": 0.04040311062490437, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.46710526315789475, + "acc_stderr": 0.040601270352363966, + "brier_score": 0.7703617335441443, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_college_mathematics": { - "acc": 0.33, - "acc_stderr": 0.04725815626252604, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.35, + "acc_stderr": 0.0479372485441102, + "brier_score": 0.785867596602279, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_conceptual_physics": { - "acc": 0.4, - "acc_stderr": 0.03202563076101735, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.3446808510638298, + "acc_stderr": 0.03106898596312215, + "brier_score": 0.9114039342379253, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_abstract_algebra": { - "acc": 0.3, - "acc_stderr": 0.046056618647183814, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "brier_score": 0.8541042943465978, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_high_school_computer_science": { - "acc": 0.41, - "acc_stderr": 0.04943110704237102, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.43, + "acc_stderr": 0.04975698519562428, + "brier_score": 0.7536844740552069, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_machine_learning": { - "acc": 0.32142857142857145, - "acc_stderr": 0.04432804055291518, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.3125, + "acc_stderr": 0.043994650575715215, + "brier_score": 0.8672657328988382, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_high_school_chemistry": { - "acc": 0.32019704433497537, - "acc_stderr": 0.032826493853041504, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.3645320197044335, + "acc_stderr": 0.03386405746062089, + "brier_score": 0.8207565586615612, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_high_school_statistics": { - "acc": 0.2824074074074074, - "acc_stderr": 0.03070137211151092, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.28703703703703703, + "acc_stderr": 0.030851992993257013, + "brier_score": 0.9131758303558689, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_elementary_mathematics": { - "acc": 0.31746031746031744, - "acc_stderr": 0.023973861998992072, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.25132275132275134, + "acc_stderr": 0.022340482339643898, + "brier_score": 0.9622714666506607, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_electrical_engineering": { - "acc": 0.47586206896551725, - "acc_stderr": 0.0416180850350153, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.43448275862068964, + "acc_stderr": 0.041307408795554966, + "brier_score": 0.7647456420359826, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "mmlu_high_school_mathematics": { - "acc": 0.2518518518518518, - "acc_stderr": 0.026466117538959912, - "timestamp": "2024-11-15T06-50-09.288337" + "acc": 0.2740740740740741, + "acc_stderr": 0.027195934804085626, + "brier_score": 0.9189945009491987, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-29-52.764283" }, "arc_challenge": { "acc": 0.5136518771331058, diff --git a/metadata/model_scores/results_meta-llama__Meta-Llama-3-8B-Instruct.json b/metadata/model_scores/results_meta-llama__Meta-Llama-3-8B-Instruct.json index 4fa3ee9..f6b91ad 100644 --- a/metadata/model_scores/results_meta-llama__Meta-Llama-3-8B-Instruct.json +++ b/metadata/model_scores/results_meta-llama__Meta-Llama-3-8B-Instruct.json @@ -1,6 +1,6 @@ { "model_name": "meta-llama/Meta-Llama-3-8B-Instruct", - "last_updated": "2024-12-04 11:23:53.204834", + "last_updated": "2024-12-19 13:39:16.941147", "results": { "harness": { "minerva_math_precalc": { @@ -228,289 +228,403 @@ "timestamp": "2024-06-12T07-18-51.438784" }, "mmlu_world_religions": { - "acc": 0.783625730994152, - "acc_stderr": 0.031581495393387324, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.7777777777777778, + "acc_stderr": 0.03188578017686398, + "brier_score": 0.3376258192694194, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_formal_logic": { - "acc": 0.49206349206349204, - "acc_stderr": 0.044715725362943486, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.5238095238095238, + "acc_stderr": 0.04467062628403273, + "brier_score": 0.6980274551083973, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_prehistory": { - "acc": 0.7407407407407407, - "acc_stderr": 0.02438366553103545, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.7283950617283951, + "acc_stderr": 0.02474862449053738, + "brier_score": 0.39291690430528736, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_moral_scenarios": { - "acc": 0.4346368715083799, - "acc_stderr": 0.01657899743549672, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.3407821229050279, + "acc_stderr": 0.01585200244986209, + "brier_score": 0.8668016282305296, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_high_school_world_history": { - "acc": 0.8438818565400844, - "acc_stderr": 0.023627159460318684, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.8354430379746836, + "acc_stderr": 0.024135736240566943, + "brier_score": 0.2441529748394776, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_moral_disputes": { - "acc": 0.7456647398843931, - "acc_stderr": 0.023445826276545546, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.6907514450867052, + "acc_stderr": 0.024883140570071755, + "brier_score": 0.4533446390696797, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_professional_law": { - "acc": 0.4784876140808344, - "acc_stderr": 0.01275841094103892, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.49022164276401564, + "acc_stderr": 0.01276779378772933, + "brier_score": 0.7291735128410968, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_logical_fallacies": { - "acc": 0.7668711656441718, - "acc_stderr": 0.0332201579577674, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.7607361963190185, + "acc_stderr": 0.03351953879521271, + "brier_score": 0.34506175696101216, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_high_school_us_history": { - "acc": 0.8529411764705882, - "acc_stderr": 0.024857478080250465, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.8186274509803921, + "acc_stderr": 0.027044621719474072, + "brier_score": 0.2588802540945586, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_philosophy": { "acc": 0.7202572347266881, "acc_stderr": 0.0254942593506949, - "timestamp": "2024-11-22T19-05-59.919138" + "brier_score": 0.4062055829570173, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_jurisprudence": { - "acc": 0.7777777777777778, - "acc_stderr": 0.0401910747255735, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.7592592592592593, + "acc_stderr": 0.041331194402438376, + "brier_score": 0.3270627816811494, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_international_law": { - "acc": 0.8181818181818182, - "acc_stderr": 0.03520893951097653, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.7603305785123967, + "acc_stderr": 0.038968789850704164, + "brier_score": 0.3526125001130658, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_high_school_european_history": { - "acc": 0.7515151515151515, - "acc_stderr": 0.03374402644139405, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.7454545454545455, + "acc_stderr": 0.03401506715249039, + "brier_score": 0.40408127337296396, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_high_school_government_and_politics": { - "acc": 0.9119170984455959, - "acc_stderr": 0.02045374660160103, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.8756476683937824, + "acc_stderr": 0.023814477086593563, + "brier_score": 0.20414191906912876, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_high_school_microeconomics": { - "acc": 0.7647058823529411, - "acc_stderr": 0.027553614467863818, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.7184873949579832, + "acc_stderr": 0.029213549414372167, + "brier_score": 0.40663777837532783, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_high_school_geography": { - "acc": 0.8434343434343434, - "acc_stderr": 0.025890520358141454, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.7777777777777778, + "acc_stderr": 0.02962022787479047, + "brier_score": 0.30299606961532244, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_high_school_psychology": { - "acc": 0.8311926605504587, - "acc_stderr": 0.01606005626853035, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.8238532110091743, + "acc_stderr": 0.01633288239343138, + "brier_score": 0.2711728027147795, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_public_relations": { - "acc": 0.6363636363636364, - "acc_stderr": 0.046075820907199756, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.6727272727272727, + "acc_stderr": 0.0449429086625209, + "brier_score": 0.4659728330390492, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_us_foreign_policy": { - "acc": 0.85, - "acc_stderr": 0.0358870281282637, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.84, + "acc_stderr": 0.03684529491774709, + "brier_score": 0.22274870775036582, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_sociology": { - "acc": 0.8706467661691543, - "acc_stderr": 0.023729830881018515, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.8557213930348259, + "acc_stderr": 0.024845753212306046, + "brier_score": 0.23113090531934954, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_high_school_macroeconomics": { - "acc": 0.658974358974359, - "acc_stderr": 0.024035489676335065, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.6461538461538462, + "acc_stderr": 0.02424378399406216, + "brier_score": 0.49839898594912685, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_security_studies": { - "acc": 0.7428571428571429, - "acc_stderr": 0.02797982353874455, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.7306122448979592, + "acc_stderr": 0.02840125202902294, + "brier_score": 0.379967944540605, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_professional_psychology": { - "acc": 0.7091503267973857, - "acc_stderr": 0.018373116915903973, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.6715686274509803, + "acc_stderr": 0.018999707383162666, + "brier_score": 0.48283691821628216, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_human_sexuality": { - "acc": 0.7786259541984732, - "acc_stderr": 0.03641297081313729, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.7633587786259542, + "acc_stderr": 0.03727673575596915, + "brier_score": 0.3548332095368855, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_econometrics": { - "acc": 0.6140350877192983, - "acc_stderr": 0.045796394220704334, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.5350877192982456, + "acc_stderr": 0.046920083813689104, + "brier_score": 0.677082317785518, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_miscellaneous": { - "acc": 0.7994891443167306, - "acc_stderr": 0.014317653708594212, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.8058748403575989, + "acc_stderr": 0.01414397027665757, + "brier_score": 0.2832781046612229, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_marketing": { - "acc": 0.905982905982906, - "acc_stderr": 0.019119892798924974, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.8931623931623932, + "acc_stderr": 0.02023714900899092, + "brier_score": 0.16389074991554042, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_management": { - "acc": 0.7766990291262136, - "acc_stderr": 0.04123553189891431, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.8252427184466019, + "acc_stderr": 0.037601780060266196, + "brier_score": 0.2718740815792001, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_nutrition": { - "acc": 0.7516339869281046, - "acc_stderr": 0.024739981355113592, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.7320261437908496, + "acc_stderr": 0.025360603796242557, + "brier_score": 0.3897167590755755, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_medical_genetics": { - "acc": 0.8, - "acc_stderr": 0.04020151261036846, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.81, + "acc_stderr": 0.03942772444036623, + "brier_score": 0.307411346256468, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_human_aging": { - "acc": 0.7309417040358744, - "acc_stderr": 0.029763779406874972, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.6771300448430493, + "acc_stderr": 0.031381476375754995, + "brier_score": 0.468792528284544, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_professional_medicine": { - "acc": 0.7242647058823529, - "acc_stderr": 0.027146271936625166, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.75, + "acc_stderr": 0.026303648393696036, + "brier_score": 0.40109802460691063, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_college_medicine": { - "acc": 0.6416184971098265, - "acc_stderr": 0.0365634365335316, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.630057803468208, + "acc_stderr": 0.0368122963339432, + "brier_score": 0.5050045005099414, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_business_ethics": { - "acc": 0.69, - "acc_stderr": 0.04648231987117316, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.65, + "acc_stderr": 0.047937248544110196, + "brier_score": 0.4819277244337173, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_clinical_knowledge": { - "acc": 0.7471698113207547, - "acc_stderr": 0.026749899771241224, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.7169811320754716, + "acc_stderr": 0.027724236492700918, + "brier_score": 0.3907958149135372, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_global_facts": { - "acc": 0.4, - "acc_stderr": 0.049236596391733084, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.38, + "acc_stderr": 0.048783173121456316, + "brier_score": 0.7258617716168976, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_virology": { - "acc": 0.5060240963855421, - "acc_stderr": 0.03892212195333045, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.5120481927710844, + "acc_stderr": 0.038913644958358175, + "brier_score": 0.8143739339624022, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_professional_accounting": { - "acc": 0.5390070921985816, - "acc_stderr": 0.029736592526424438, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.549645390070922, + "acc_stderr": 0.02968010556502904, + "brier_score": 0.6166727913872267, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_college_physics": { - "acc": 0.5098039215686274, - "acc_stderr": 0.04974229460422817, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.5, + "acc_stderr": 0.04975185951049946, + "brier_score": 0.669272266134827, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_high_school_physics": { - "acc": 0.44370860927152317, - "acc_stderr": 0.04056527902281732, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.4304635761589404, + "acc_stderr": 0.04042809961395634, + "brier_score": 0.7136826066672557, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_high_school_biology": { - "acc": 0.7806451612903226, - "acc_stderr": 0.023540799358723302, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.7645161290322581, + "acc_stderr": 0.024137632429337703, + "brier_score": 0.36284198711840915, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_college_biology": { - "acc": 0.7916666666666666, - "acc_stderr": 0.03396116205845334, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.75, + "acc_stderr": 0.03621034121889507, + "brier_score": 0.3503528186466499, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_anatomy": { "acc": 0.6370370370370371, - "acc_stderr": 0.041539484047424, - "timestamp": "2024-11-22T19-05-59.919138" + "acc_stderr": 0.04153948404742399, + "brier_score": 0.5389026229745647, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_college_chemistry": { - "acc": 0.47, - "acc_stderr": 0.05016135580465919, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.42, + "acc_stderr": 0.049604496374885836, + "brier_score": 0.7118320327181862, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_computer_security": { "acc": 0.77, - "acc_stderr": 0.042295258468165065, - "timestamp": "2024-11-22T19-05-59.919138" + "acc_stderr": 0.04229525846816506, + "brier_score": 0.37266458073179076, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_college_computer_science": { - "acc": 0.58, - "acc_stderr": 0.049604496374885836, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.51, + "acc_stderr": 0.05024183937956911, + "brier_score": 0.6364190283622218, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_astronomy": { - "acc": 0.6973684210526315, - "acc_stderr": 0.03738520676119667, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.6842105263157895, + "acc_stderr": 0.03782728980865469, + "brier_score": 0.4362631512301302, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_college_mathematics": { - "acc": 0.38, - "acc_stderr": 0.04878317312145633, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.33, + "acc_stderr": 0.04725815626252604, + "brier_score": 0.7951255964154708, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_conceptual_physics": { - "acc": 0.6042553191489362, - "acc_stderr": 0.031967586978353627, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.5361702127659574, + "acc_stderr": 0.032600385118357715, + "brier_score": 0.6263913485195526, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_abstract_algebra": { - "acc": 0.32, - "acc_stderr": 0.046882617226215034, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "brier_score": 0.7972656381802152, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_high_school_computer_science": { - "acc": 0.75, - "acc_stderr": 0.04351941398892446, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.69, + "acc_stderr": 0.04648231987117316, + "brier_score": 0.4321421643505633, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_machine_learning": { - "acc": 0.5446428571428571, - "acc_stderr": 0.04726835553719097, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.48214285714285715, + "acc_stderr": 0.047427623612430116, + "brier_score": 0.6323848144772048, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_high_school_chemistry": { - "acc": 0.5024630541871922, - "acc_stderr": 0.03517945038691063, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.4630541871921182, + "acc_stderr": 0.035083705204426656, + "brier_score": 0.7004761750941393, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_high_school_statistics": { - "acc": 0.5370370370370371, - "acc_stderr": 0.03400603625538271, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.5185185185185185, + "acc_stderr": 0.03407632093854052, + "brier_score": 0.6747863159719197, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_elementary_mathematics": { - "acc": 0.4470899470899471, - "acc_stderr": 0.025606723995777025, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.455026455026455, + "acc_stderr": 0.025646928361049398, + "brier_score": 0.7064055129748947, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_electrical_engineering": { "acc": 0.6275862068965518, - "acc_stderr": 0.04028731532947559, - "timestamp": "2024-11-22T19-05-59.919138" + "acc_stderr": 0.04028731532947558, + "brier_score": 0.5380762418986331, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "mmlu_high_school_mathematics": { - "acc": 0.3925925925925926, - "acc_stderr": 0.02977384701253297, - "timestamp": "2024-11-22T19-05-59.919138" + "acc": 0.3888888888888889, + "acc_stderr": 0.029723278961476668, + "brier_score": 0.7657337575523382, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T20-07-57.894580" }, "arc_challenge": { "acc": 0.5767918088737202, diff --git a/metadata/model_scores/results_microsoft__phi-1_5.json b/metadata/model_scores/results_microsoft__phi-1_5.json index ffad97d..898a635 100644 --- a/metadata/model_scores/results_microsoft__phi-1_5.json +++ b/metadata/model_scores/results_microsoft__phi-1_5.json @@ -1,292 +1,406 @@ { "model_name": "microsoft__phi-1_5", - "last_updated": "2024-12-04 11:25:53.081637", + "last_updated": "2024-12-19 13:42:10.917708", "results": { "harness": { "mmlu_world_religions": { - "acc": 0.4502923976608187, - "acc_stderr": 0.03815827365913236, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.42105263157894735, + "acc_stderr": 0.037867207062342145, + "brier_score": 0.7360008752497482, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_formal_logic": { - "acc": 0.2698412698412698, - "acc_stderr": 0.03970158273235173, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.2777777777777778, + "acc_stderr": 0.040061680838488774, + "brier_score": 0.777206013151806, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_prehistory": { - "acc": 0.4228395061728395, - "acc_stderr": 0.027487472980871605, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.4166666666666667, + "acc_stderr": 0.02743162372241501, + "brier_score": 0.7380412963405232, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_moral_scenarios": { - "acc": 0.24692737430167597, - "acc_stderr": 0.014422292204808843, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.21899441340782122, + "acc_stderr": 0.013831676687303195, + "brier_score": 0.7901934382936933, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_high_school_world_history": { - "acc": 0.510548523206751, - "acc_stderr": 0.032539983791662855, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.5485232067510548, + "acc_stderr": 0.032393600173974704, + "brier_score": 0.5792982228967096, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_moral_disputes": { - "acc": 0.5491329479768786, - "acc_stderr": 0.026788811931562753, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.5173410404624278, + "acc_stderr": 0.026902900458666647, + "brier_score": 0.6423391106948374, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_professional_law": { - "acc": 0.34028683181225555, - "acc_stderr": 0.012101217610223773, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.3318122555410691, + "acc_stderr": 0.012026088259897623, + "brier_score": 0.7996198436303776, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_logical_fallacies": { - "acc": 0.5276073619631901, - "acc_stderr": 0.0392237829061099, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.50920245398773, + "acc_stderr": 0.03927705600787443, + "brier_score": 0.6745861358113816, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_high_school_us_history": { - "acc": 0.46078431372549017, - "acc_stderr": 0.03498501649369527, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.49019607843137253, + "acc_stderr": 0.035086373586305716, + "brier_score": 0.6738082218665467, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_philosophy": { - "acc": 0.4790996784565916, - "acc_stderr": 0.028373270961069414, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.4662379421221865, + "acc_stderr": 0.0283332771095628, + "brier_score": 0.7074817938782464, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_jurisprudence": { - "acc": 0.5185185185185185, - "acc_stderr": 0.04830366024635331, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.5555555555555556, + "acc_stderr": 0.04803752235190193, + "brier_score": 0.6202091131314537, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_international_law": { - "acc": 0.6033057851239669, - "acc_stderr": 0.04465869780531009, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.6363636363636364, + "acc_stderr": 0.043913262867240704, + "brier_score": 0.515450487912183, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_high_school_european_history": { - "acc": 0.48484848484848486, - "acc_stderr": 0.039025510073744475, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.5212121212121212, + "acc_stderr": 0.03900828913737301, + "brier_score": 0.6271142903227849, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_high_school_government_and_politics": { - "acc": 0.538860103626943, - "acc_stderr": 0.035975244117345775, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.5492227979274611, + "acc_stderr": 0.03590910952235524, + "brier_score": 0.6016571539143739, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_high_school_microeconomics": { - "acc": 0.453781512605042, - "acc_stderr": 0.03233943468182088, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.4327731092436975, + "acc_stderr": 0.03218358107742613, + "brier_score": 0.6787214859797679, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_high_school_geography": { "acc": 0.5202020202020202, - "acc_stderr": 0.03559443565563919, - "timestamp": "2024-11-10T18-48-26.589447" + "acc_stderr": 0.03559443565563918, + "brier_score": 0.6260096931634569, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_high_school_psychology": { - "acc": 0.5724770642201835, - "acc_stderr": 0.021210910204300434, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.5779816513761468, + "acc_stderr": 0.021174991407763175, + "brier_score": 0.5846518035857755, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_public_relations": { - "acc": 0.5181818181818182, - "acc_stderr": 0.04785964010794916, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.509090909090909, + "acc_stderr": 0.0478833976870286, + "brier_score": 0.6436576965735664, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_us_foreign_policy": { - "acc": 0.65, - "acc_stderr": 0.047937248544110196, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.64, + "acc_stderr": 0.048241815132442176, + "brier_score": 0.5065862216963781, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_sociology": { - "acc": 0.6417910447761194, - "acc_stderr": 0.03390393042268814, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.6716417910447762, + "acc_stderr": 0.033206858897443244, + "brier_score": 0.4555626101107058, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_high_school_macroeconomics": { - "acc": 0.4282051282051282, - "acc_stderr": 0.02508830145469483, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.4153846153846154, + "acc_stderr": 0.02498535492310234, + "brier_score": 0.7287925023449086, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_security_studies": { - "acc": 0.5061224489795918, - "acc_stderr": 0.03200682020163908, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.4775510204081633, + "acc_stderr": 0.03197694118713672, + "brier_score": 0.6596316885420239, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_professional_psychology": { - "acc": 0.4019607843137255, - "acc_stderr": 0.01983517648437538, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.39705882352941174, + "acc_stderr": 0.01979448890002411, + "brier_score": 0.7788372440904848, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_human_sexuality": { - "acc": 0.4732824427480916, - "acc_stderr": 0.04379024936553894, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.5114503816793893, + "acc_stderr": 0.043841400240780176, + "brier_score": 0.6196890231613316, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_econometrics": { - "acc": 0.23684210526315788, - "acc_stderr": 0.039994238792813365, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.2719298245614035, + "acc_stderr": 0.04185774424022056, + "brier_score": 0.8722626035349442, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_miscellaneous": { - "acc": 0.508301404853129, - "acc_stderr": 0.017877498991072, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.5057471264367817, + "acc_stderr": 0.01787878232612923, + "brier_score": 0.6169403423513001, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_marketing": { - "acc": 0.7008547008547008, - "acc_stderr": 0.029996951858349483, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.6794871794871795, + "acc_stderr": 0.03057281131029961, + "brier_score": 0.4510237014621223, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_management": { - "acc": 0.5825242718446602, - "acc_stderr": 0.04882840548212238, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.6116504854368932, + "acc_stderr": 0.04825729337356389, + "brier_score": 0.5332099122676459, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_nutrition": { - "acc": 0.5032679738562091, - "acc_stderr": 0.028629305194003533, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.5163398692810458, + "acc_stderr": 0.028614624752805434, + "brier_score": 0.6350133525853426, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_medical_genetics": { - "acc": 0.5, - "acc_stderr": 0.050251890762960605, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.44, + "acc_stderr": 0.049888765156985884, + "brier_score": 0.6881471618921275, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_human_aging": { - "acc": 0.484304932735426, - "acc_stderr": 0.0335412657542081, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.45739910313901344, + "acc_stderr": 0.033435777055830646, + "brier_score": 0.7105188295097965, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_professional_medicine": { - "acc": 0.33455882352941174, - "acc_stderr": 0.02866199620233531, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.3272058823529412, + "acc_stderr": 0.028501452860396563, + "brier_score": 0.8125044459404857, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_college_medicine": { - "acc": 0.4161849710982659, - "acc_stderr": 0.037585177754049466, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.3930635838150289, + "acc_stderr": 0.0372424959581773, + "brier_score": 0.763439234194539, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_business_ethics": { - "acc": 0.51, - "acc_stderr": 0.05024183937956912, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.5, + "acc_stderr": 0.050251890762960605, + "brier_score": 0.6084030677575967, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_clinical_knowledge": { - "acc": 0.49433962264150944, - "acc_stderr": 0.03077090076385131, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.4641509433962264, + "acc_stderr": 0.030693675018458006, + "brier_score": 0.7057383144243409, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_global_facts": { - "acc": 0.32, - "acc_stderr": 0.04688261722621504, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.24, + "acc_stderr": 0.042923469599092816, + "brier_score": 0.7998431789880189, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_virology": { "acc": 0.41566265060240964, - "acc_stderr": 0.03836722176598052, - "timestamp": "2024-11-10T18-48-26.589447" + "acc_stderr": 0.038367221765980515, + "brier_score": 0.7769035591355941, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_professional_accounting": { - "acc": 0.3049645390070922, - "acc_stderr": 0.02746470844202214, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.2907801418439716, + "acc_stderr": 0.027090664368353178, + "brier_score": 0.8232895523294619, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_college_physics": { - "acc": 0.29411764705882354, - "acc_stderr": 0.04533838195929775, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.30392156862745096, + "acc_stderr": 0.045766654032077636, + "brier_score": 0.8065082489214286, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_high_school_physics": { - "acc": 0.2781456953642384, - "acc_stderr": 0.03658603262763743, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.31788079470198677, + "acc_stderr": 0.03802039760107903, + "brier_score": 0.7968941338535283, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_high_school_biology": { - "acc": 0.4483870967741935, - "acc_stderr": 0.028292056830112728, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.47419354838709676, + "acc_stderr": 0.02840609505765332, + "brier_score": 0.6673893585036331, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_college_biology": { - "acc": 0.375, - "acc_stderr": 0.04048439222695598, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.3541666666666667, + "acc_stderr": 0.039994111357535424, + "brier_score": 0.7600764132559394, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_anatomy": { - "acc": 0.4666666666666667, - "acc_stderr": 0.043097329010363554, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.4222222222222222, + "acc_stderr": 0.04266763404099582, + "brier_score": 0.7251975156324822, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_college_chemistry": { "acc": 0.27, - "acc_stderr": 0.044619604333847394, - "timestamp": "2024-11-10T18-48-26.589447" + "acc_stderr": 0.0446196043338474, + "brier_score": 0.7605508640162124, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_computer_security": { - "acc": 0.53, - "acc_stderr": 0.05016135580465919, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.5, + "acc_stderr": 0.050251890762960605, + "brier_score": 0.685223148320539, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_college_computer_science": { - "acc": 0.46, - "acc_stderr": 0.05009082659620333, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.37, + "acc_stderr": 0.048523658709391, + "brier_score": 0.7721012067814597, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_astronomy": { - "acc": 0.4144736842105263, - "acc_stderr": 0.04008973785779206, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.39473684210526316, + "acc_stderr": 0.039777499346220734, + "brier_score": 0.7257436739204995, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_college_mathematics": { - "acc": 0.41, - "acc_stderr": 0.04943110704237101, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.37, + "acc_stderr": 0.048523658709391, + "brier_score": 0.7138852319996118, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_conceptual_physics": { - "acc": 0.37446808510638296, - "acc_stderr": 0.031639106653672915, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.3659574468085106, + "acc_stderr": 0.031489558297455304, + "brier_score": 0.8308913625317457, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_abstract_algebra": { - "acc": 0.36, - "acc_stderr": 0.048241815132442176, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "brier_score": 0.8358822212811206, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_high_school_computer_science": { - "acc": 0.46, - "acc_stderr": 0.05009082659620333, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.49, + "acc_stderr": 0.05024183937956911, + "brier_score": 0.6243291870440001, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_machine_learning": { "acc": 0.39285714285714285, "acc_stderr": 0.04635550135609976, - "timestamp": "2024-11-10T18-48-26.589447" + "brier_score": 0.7403036536446453, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_high_school_chemistry": { - "acc": 0.3448275862068966, - "acc_stderr": 0.03344283744280458, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.33004926108374383, + "acc_stderr": 0.03308530426228258, + "brier_score": 0.7808990504625262, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_high_school_statistics": { - "acc": 0.28703703703703703, - "acc_stderr": 0.030851992993257013, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.26851851851851855, + "acc_stderr": 0.030225226160012393, + "brier_score": 0.8587462682397424, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_elementary_mathematics": { - "acc": 0.30423280423280424, - "acc_stderr": 0.023695415009463084, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.30952380952380953, + "acc_stderr": 0.023809523809523867, + "brier_score": 0.7900592112702733, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_electrical_engineering": { - "acc": 0.4689655172413793, - "acc_stderr": 0.04158632762097828, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.46206896551724136, + "acc_stderr": 0.041546596717075474, + "brier_score": 0.6533778994999127, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "mmlu_high_school_mathematics": { - "acc": 0.23333333333333334, - "acc_stderr": 0.02578787422095931, - "timestamp": "2024-11-10T18-48-26.589447" + "acc": 0.24444444444444444, + "acc_stderr": 0.02620276653465215, + "brier_score": 0.7963735663036272, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-10-38.843036" }, "arc_challenge": { "acc": 0.5008532423208191, diff --git a/metadata/model_scores/results_microsoft__phi-2.json b/metadata/model_scores/results_microsoft__phi-2.json index a900e11..0b6a911 100644 --- a/metadata/model_scores/results_microsoft__phi-2.json +++ b/metadata/model_scores/results_microsoft__phi-2.json @@ -1,292 +1,406 @@ { "model_name": "microsoft__phi-2", - "last_updated": "2024-12-04 11:24:13.297237", + "last_updated": "2024-12-19 13:40:01.459180", "results": { "harness": { "mmlu_world_religions": { - "acc": 0.6900584795321637, - "acc_stderr": 0.03546976959393163, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.695906432748538, + "acc_stderr": 0.03528211258245231, + "brier_score": 0.4124207429640924, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_formal_logic": { - "acc": 0.35714285714285715, - "acc_stderr": 0.04285714285714281, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.3333333333333333, + "acc_stderr": 0.04216370213557835, + "brier_score": 0.7220382999831597, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_prehistory": { - "acc": 0.6234567901234568, - "acc_stderr": 0.026959344518747784, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.6018518518518519, + "acc_stderr": 0.02723741509459247, + "brier_score": 0.5089921216509934, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_moral_scenarios": { - "acc": 0.3139664804469274, - "acc_stderr": 0.015521923933523647, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.2335195530726257, + "acc_stderr": 0.01414957534897626, + "brier_score": 0.8788760175524666, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_high_school_world_history": { - "acc": 0.7383966244725738, - "acc_stderr": 0.028609516716994934, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.7552742616033755, + "acc_stderr": 0.027985699387036416, + "brier_score": 0.42015101380924585, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_moral_disputes": { - "acc": 0.6676300578034682, - "acc_stderr": 0.025361168749688214, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.653179190751445, + "acc_stderr": 0.025624723994030464, + "brier_score": 0.4953851144132304, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_professional_law": { - "acc": 0.4198174706649283, - "acc_stderr": 0.01260496081608737, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.39895697522816165, + "acc_stderr": 0.01250675765529367, + "brier_score": 0.6969188345540653, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_logical_fallacies": { - "acc": 0.7484662576687117, - "acc_stderr": 0.034089978868575295, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.7607361963190185, + "acc_stderr": 0.033519538795212696, + "brier_score": 0.3697065462256061, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_high_school_us_history": { - "acc": 0.6617647058823529, - "acc_stderr": 0.03320574612945431, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.6813725490196079, + "acc_stderr": 0.0327028718148208, + "brier_score": 0.4565368516589864, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_philosophy": { - "acc": 0.6270096463022508, - "acc_stderr": 0.027466610213140112, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.5755627009646302, + "acc_stderr": 0.028071928247946205, + "brier_score": 0.5163780448328548, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_jurisprudence": { - "acc": 0.7129629629629629, - "acc_stderr": 0.043733130409147614, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.7037037037037037, + "acc_stderr": 0.04414343666854933, + "brier_score": 0.40901176900233016, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_international_law": { - "acc": 0.743801652892562, - "acc_stderr": 0.03984979653302872, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.7603305785123967, + "acc_stderr": 0.038968789850704164, + "brier_score": 0.35251137375069447, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_high_school_european_history": { - "acc": 0.6424242424242425, - "acc_stderr": 0.037425970438065864, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.703030303030303, + "acc_stderr": 0.03567969772268049, + "brier_score": 0.4580662616057771, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_high_school_government_and_politics": { - "acc": 0.8082901554404145, - "acc_stderr": 0.02840895362624526, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.7668393782383419, + "acc_stderr": 0.03051611137147602, + "brier_score": 0.32228783313694453, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_high_school_microeconomics": { - "acc": 0.6050420168067226, - "acc_stderr": 0.03175367846096626, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.5882352941176471, + "acc_stderr": 0.03196876989195778, + "brier_score": 0.5115888159559108, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_high_school_geography": { - "acc": 0.7373737373737373, - "acc_stderr": 0.03135305009533085, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.7222222222222222, + "acc_stderr": 0.03191178226713547, + "brier_score": 0.38625008579175607, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_high_school_psychology": { - "acc": 0.8, - "acc_stderr": 0.01714985851425093, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.7651376146788991, + "acc_stderr": 0.018175110510343578, + "brier_score": 0.3279035759824191, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_public_relations": { - "acc": 0.6727272727272727, - "acc_stderr": 0.0449429086625209, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.6454545454545455, + "acc_stderr": 0.045820048415054174, + "brier_score": 0.45567858680867424, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_us_foreign_policy": { - "acc": 0.76, - "acc_stderr": 0.04292346959909283, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.72, + "acc_stderr": 0.045126085985421276, + "brier_score": 0.32462247031549857, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_sociology": { - "acc": 0.8109452736318408, - "acc_stderr": 0.027686913588013028, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.7810945273631841, + "acc_stderr": 0.029239174636647, + "brier_score": 0.30534825640524377, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_high_school_macroeconomics": { - "acc": 0.5692307692307692, - "acc_stderr": 0.02510682066053975, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.5769230769230769, + "acc_stderr": 0.02504919787604235, + "brier_score": 0.5302869478068375, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_security_studies": { - "acc": 0.7306122448979592, - "acc_stderr": 0.02840125202902294, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.6571428571428571, + "acc_stderr": 0.03038726291954773, + "brier_score": 0.4391358426536197, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_professional_psychology": { - "acc": 0.5604575163398693, - "acc_stderr": 0.02007942040808792, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.5392156862745098, + "acc_stderr": 0.02016552331390791, + "brier_score": 0.5711487339018314, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_human_sexuality": { - "acc": 0.7099236641221374, - "acc_stderr": 0.03980066246467765, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.6564885496183206, + "acc_stderr": 0.041649760719448786, + "brier_score": 0.45412460592973525, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_econometrics": { - "acc": 0.38596491228070173, - "acc_stderr": 0.045796394220704334, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.2982456140350877, + "acc_stderr": 0.04303684033537314, + "brier_score": 0.7754655884085445, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_miscellaneous": { - "acc": 0.6896551724137931, - "acc_stderr": 0.016543785026048318, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.6871008939974457, + "acc_stderr": 0.016580935940304048, + "brier_score": 0.40109685014778523, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_marketing": { - "acc": 0.8205128205128205, - "acc_stderr": 0.025140935950335442, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.7948717948717948, + "acc_stderr": 0.02645350805404033, + "brier_score": 0.27575313282111125, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_management": { "acc": 0.7378640776699029, "acc_stderr": 0.04354631077260595, - "timestamp": "2024-11-20T12-19-35.462797" + "brier_score": 0.3819982905394013, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_nutrition": { - "acc": 0.6176470588235294, - "acc_stderr": 0.027826109307283683, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.6111111111111112, + "acc_stderr": 0.02791405551046801, + "brier_score": 0.4926628596822712, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_medical_genetics": { "acc": 0.63, - "acc_stderr": 0.04852365870939099, - "timestamp": "2024-11-20T12-19-35.462797" + "acc_stderr": 0.048523658709391, + "brier_score": 0.48573602332047644, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_human_aging": { - "acc": 0.6502242152466368, - "acc_stderr": 0.03200736719484503, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.6367713004484304, + "acc_stderr": 0.032277904428505, + "brier_score": 0.49588156699058694, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_professional_medicine": { - "acc": 0.47794117647058826, - "acc_stderr": 0.03034326422421352, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.4485294117647059, + "acc_stderr": 0.030211479609121593, + "brier_score": 0.6606940817072791, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_college_medicine": { - "acc": 0.5953757225433526, - "acc_stderr": 0.03742461193887248, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.5260115606936416, + "acc_stderr": 0.03807301726504511, + "brier_score": 0.5745741819956182, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_business_ethics": { - "acc": 0.6, - "acc_stderr": 0.049236596391733084, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.58, + "acc_stderr": 0.049604496374885836, + "brier_score": 0.4980841866410591, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_clinical_knowledge": { - "acc": 0.6113207547169811, - "acc_stderr": 0.030000485448675986, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.6188679245283019, + "acc_stderr": 0.029890609686286644, + "brier_score": 0.4830551320075802, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_global_facts": { - "acc": 0.34, - "acc_stderr": 0.04760952285695235, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.36, + "acc_stderr": 0.04824181513244218, + "brier_score": 0.7370184276518171, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_virology": { - "acc": 0.4759036144578313, - "acc_stderr": 0.03887971849597264, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.4819277108433735, + "acc_stderr": 0.03889951252827216, + "brier_score": 0.6755955415876875, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_professional_accounting": { - "acc": 0.4432624113475177, - "acc_stderr": 0.029634838473766006, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.42907801418439717, + "acc_stderr": 0.029525914302558555, + "brier_score": 0.6734628820426131, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_college_physics": { - "acc": 0.3627450980392157, - "acc_stderr": 0.04784060704105654, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.3137254901960784, + "acc_stderr": 0.04617034827006718, + "brier_score": 0.746275000406364, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_high_school_physics": { - "acc": 0.37748344370860926, - "acc_stderr": 0.0395802723112157, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.3973509933774834, + "acc_stderr": 0.0399552400768168, + "brier_score": 0.7383930383603438, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_high_school_biology": { - "acc": 0.7, - "acc_stderr": 0.02606936229533514, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.6838709677419355, + "acc_stderr": 0.026450874489042767, + "brier_score": 0.43640200672949025, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_college_biology": { - "acc": 0.6666666666666666, - "acc_stderr": 0.039420826399272135, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.6111111111111112, + "acc_stderr": 0.04076663253918567, + "brier_score": 0.4879789821016725, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_anatomy": { - "acc": 0.4444444444444444, - "acc_stderr": 0.04292596718256981, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.45185185185185184, + "acc_stderr": 0.04299268905480864, + "brier_score": 0.6026312492166612, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_college_chemistry": { - "acc": 0.41, - "acc_stderr": 0.049431107042371025, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.37, + "acc_stderr": 0.04852365870939099, + "brier_score": 0.6882345069402325, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_computer_security": { - "acc": 0.74, - "acc_stderr": 0.044084400227680794, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.64, + "acc_stderr": 0.04824181513244218, + "brier_score": 0.4518442546275091, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_college_computer_science": { - "acc": 0.43, - "acc_stderr": 0.04975698519562428, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.47, + "acc_stderr": 0.05016135580465919, + "brier_score": 0.6820303010165037, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_astronomy": { - "acc": 0.5855263157894737, - "acc_stderr": 0.04008973785779206, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.5592105263157895, + "acc_stderr": 0.04040311062490436, + "brier_score": 0.5555860131240212, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_college_mathematics": { - "acc": 0.38, - "acc_stderr": 0.04878317312145632, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.44, + "acc_stderr": 0.04988876515698589, + "brier_score": 0.7020125283359032, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_conceptual_physics": { - "acc": 0.5063829787234042, - "acc_stderr": 0.032683358999363366, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.4765957446808511, + "acc_stderr": 0.03265019475033582, + "brier_score": 0.632215822853458, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_abstract_algebra": { "acc": 0.31, "acc_stderr": 0.04648231987117316, - "timestamp": "2024-11-20T12-19-35.462797" + "brier_score": 0.7463719598724317, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_high_school_computer_science": { - "acc": 0.64, - "acc_stderr": 0.04824181513244218, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.71, + "acc_stderr": 0.045604802157206845, + "brier_score": 0.43476423654292357, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_machine_learning": { "acc": 0.49107142857142855, "acc_stderr": 0.04745033255489123, - "timestamp": "2024-11-20T12-19-35.462797" + "brier_score": 0.616295780702097, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_high_school_chemistry": { - "acc": 0.4827586206896552, - "acc_stderr": 0.035158955511657, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.46798029556650245, + "acc_stderr": 0.03510766597959217, + "brier_score": 0.6137650704809741, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_high_school_statistics": { - "acc": 0.49537037037037035, - "acc_stderr": 0.03409825519163572, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.4398148148148148, + "acc_stderr": 0.033851779760448106, + "brier_score": 0.6910018927397092, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_elementary_mathematics": { - "acc": 0.455026455026455, - "acc_stderr": 0.02564692836104939, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.3862433862433862, + "acc_stderr": 0.025075981767601688, + "brier_score": 0.6964182032495722, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_electrical_engineering": { "acc": 0.5448275862068965, "acc_stderr": 0.04149886942192117, - "timestamp": "2024-11-20T12-19-35.462797" + "brier_score": 0.608615043854258, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "mmlu_high_school_mathematics": { - "acc": 0.3333333333333333, - "acc_stderr": 0.028742040903948485, - "timestamp": "2024-11-20T12-19-35.462797" + "acc": 0.29259259259259257, + "acc_stderr": 0.027738969632176088, + "brier_score": 0.7662039210743501, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-14-12.511021" }, "arc_challenge": { "acc": 0.575938566552901, diff --git a/metadata/model_scores/results_mosaicml__mpt-7b-instruct.json b/metadata/model_scores/results_mosaicml__mpt-7b-instruct.json index 7eca117..3a6b8b1 100644 --- a/metadata/model_scores/results_mosaicml__mpt-7b-instruct.json +++ b/metadata/model_scores/results_mosaicml__mpt-7b-instruct.json @@ -1,6 +1,6 @@ { "model_name": "mosaicml/mpt-7b-instruct", - "last_updated": "2024-12-04 11:23:16.685263", + "last_updated": "2024-12-19 13:38:36.368712", "results": { "harness": { "minerva_math_precalc": { @@ -228,289 +228,403 @@ "timestamp": "2024-06-09T17-10-00.585190" }, "mmlu_world_religions": { - "acc": 0.2222222222222222, - "acc_stderr": 0.03188578017686398, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.4269005847953216, + "acc_stderr": 0.03793620616529918, + "brier_score": 0.7257681021251722, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_formal_logic": { - "acc": 0.20634920634920634, - "acc_stderr": 0.03619604524124252, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.2619047619047619, + "acc_stderr": 0.039325376803928704, + "brier_score": 0.7638751497517091, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_prehistory": { - "acc": 0.32407407407407407, - "acc_stderr": 0.02604176620271716, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.35185185185185186, + "acc_stderr": 0.02657148348071997, + "brier_score": 0.7306776171304961, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_moral_scenarios": { - "acc": 0.24134078212290502, - "acc_stderr": 0.01431099954796146, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.2581005586592179, + "acc_stderr": 0.014635185616527829, + "brier_score": 0.7662986423904314, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_high_school_world_history": { - "acc": 0.24472573839662448, - "acc_stderr": 0.02798569938703643, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.33755274261603374, + "acc_stderr": 0.03078154910202623, + "brier_score": 0.7421268302824199, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_moral_disputes": { - "acc": 0.3208092485549133, - "acc_stderr": 0.025131000233647904, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.33236994219653176, + "acc_stderr": 0.02536116874968821, + "brier_score": 0.7561425416002233, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_professional_law": { - "acc": 0.2842242503259452, - "acc_stderr": 0.011519880596516078, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.3057366362451108, + "acc_stderr": 0.011766973847072917, + "brier_score": 0.7542538520161718, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_logical_fallacies": { - "acc": 0.3128834355828221, - "acc_stderr": 0.036429145782924055, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.25766871165644173, + "acc_stderr": 0.03436150827846917, + "brier_score": 0.8284424973497988, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_high_school_us_history": { - "acc": 0.24509803921568626, - "acc_stderr": 0.030190282453501954, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.31862745098039214, + "acc_stderr": 0.032702871814820796, + "brier_score": 0.7372789272582697, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_philosophy": { - "acc": 0.31511254019292606, - "acc_stderr": 0.026385273703464492, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.39228295819935693, + "acc_stderr": 0.027731258647012, + "brier_score": 0.721029565430574, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_jurisprudence": { - "acc": 0.3148148148148148, - "acc_stderr": 0.04489931073591311, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.37962962962962965, + "acc_stderr": 0.04691521224077742, + "brier_score": 0.726041299898626, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_international_law": { - "acc": 0.32231404958677684, - "acc_stderr": 0.042664163633521664, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.33884297520661155, + "acc_stderr": 0.0432076780753667, + "brier_score": 0.7047682053011297, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_high_school_european_history": { - "acc": 0.2909090909090909, - "acc_stderr": 0.03546563019624336, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.3696969696969697, + "acc_stderr": 0.03769430314512568, + "brier_score": 0.7132406579401299, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_high_school_government_and_politics": { - "acc": 0.3626943005181347, - "acc_stderr": 0.03469713791704372, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.46113989637305697, + "acc_stderr": 0.03597524411734578, + "brier_score": 0.6654367489833136, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_high_school_microeconomics": { - "acc": 0.3319327731092437, - "acc_stderr": 0.030588697013783667, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.3739495798319328, + "acc_stderr": 0.031429466378837076, + "brier_score": 0.7127620035581791, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_high_school_geography": { - "acc": 0.3434343434343434, - "acc_stderr": 0.03383201223244443, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.3686868686868687, + "acc_stderr": 0.034373055019806184, + "brier_score": 0.7379190244649863, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_high_school_psychology": { - "acc": 0.3321100917431193, - "acc_stderr": 0.020192682985423357, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.3431192660550459, + "acc_stderr": 0.02035477773608604, + "brier_score": 0.7530211886673183, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_public_relations": { - "acc": 0.37272727272727274, - "acc_stderr": 0.04631381319425464, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.39090909090909093, + "acc_stderr": 0.04673752333670239, + "brier_score": 0.7280283275482469, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_us_foreign_policy": { - "acc": 0.36, - "acc_stderr": 0.04824181513244218, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.44, + "acc_stderr": 0.049888765156985884, + "brier_score": 0.6790437427997017, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_sociology": { - "acc": 0.24875621890547264, - "acc_stderr": 0.030567675938916707, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.3880597014925373, + "acc_stderr": 0.0344578996436275, + "brier_score": 0.6893012864485502, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_high_school_macroeconomics": { - "acc": 0.35384615384615387, - "acc_stderr": 0.024243783994062153, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.3333333333333333, + "acc_stderr": 0.023901157979402544, + "brier_score": 0.7434676816538845, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_security_studies": { "acc": 0.42857142857142855, - "acc_stderr": 0.031680911612338825, - "timestamp": "2024-11-07T02-15-50.682989" + "acc_stderr": 0.03168091161233883, + "brier_score": 0.7022607912797364, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_professional_psychology": { - "acc": 0.315359477124183, - "acc_stderr": 0.018798086284886887, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.30392156862745096, + "acc_stderr": 0.018607552131279827, + "brier_score": 0.7529173430663462, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_human_sexuality": { - "acc": 0.3969465648854962, - "acc_stderr": 0.04291135671009224, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.3511450381679389, + "acc_stderr": 0.04186445163013751, + "brier_score": 0.7067376678367041, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_econometrics": { - "acc": 0.2719298245614035, - "acc_stderr": 0.04185774424022056, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.2982456140350877, + "acc_stderr": 0.04303684033537315, + "brier_score": 0.8098249857307726, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_miscellaneous": { - "acc": 0.334610472541507, - "acc_stderr": 0.016873468641592157, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.438058748403576, + "acc_stderr": 0.017742232238257237, + "brier_score": 0.6926811262238252, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_marketing": { - "acc": 0.33760683760683763, - "acc_stderr": 0.030980296992618558, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.42735042735042733, + "acc_stderr": 0.03240847393516327, + "brier_score": 0.6712686523189929, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_management": { - "acc": 0.27184466019417475, - "acc_stderr": 0.044052680241409216, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.2912621359223301, + "acc_stderr": 0.04498676320572922, + "brier_score": 0.7691905852991028, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_nutrition": { "acc": 0.31699346405228757, "acc_stderr": 0.02664327847450875, - "timestamp": "2024-11-07T02-15-50.682989" + "brier_score": 0.7576935523056313, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_medical_genetics": { "acc": 0.37, - "acc_stderr": 0.04852365870939099, - "timestamp": "2024-11-07T02-15-50.682989" + "acc_stderr": 0.048523658709391, + "brier_score": 0.7319577359874132, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_human_aging": { - "acc": 0.34080717488789236, - "acc_stderr": 0.031811497470553604, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.42152466367713004, + "acc_stderr": 0.033141902221106564, + "brier_score": 0.6962093368385792, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_professional_medicine": { - "acc": 0.23897058823529413, - "acc_stderr": 0.025905280644893006, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.25, + "acc_stderr": 0.026303648393696036, + "brier_score": 0.7660407492028689, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_college_medicine": { - "acc": 0.3179190751445087, - "acc_stderr": 0.03550683989165582, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.3063583815028902, + "acc_stderr": 0.03514942551267437, + "brier_score": 0.7834385880245865, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_business_ethics": { - "acc": 0.29, - "acc_stderr": 0.04560480215720684, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.41, + "acc_stderr": 0.049431107042371025, + "brier_score": 0.7095796975594699, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_clinical_knowledge": { - "acc": 0.35471698113207545, - "acc_stderr": 0.02944517532819959, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.35094339622641507, + "acc_stderr": 0.02937364625323469, + "brier_score": 0.7387591157712322, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_global_facts": { - "acc": 0.29, - "acc_stderr": 0.045604802157206845, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.36, + "acc_stderr": 0.04824181513244218, + "brier_score": 0.7227671849826451, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_virology": { - "acc": 0.3614457831325301, - "acc_stderr": 0.037400593820293204, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.42168674698795183, + "acc_stderr": 0.03844453181770917, + "brier_score": 0.7084750245331242, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_professional_accounting": { - "acc": 0.25177304964539005, - "acc_stderr": 0.025892151156709405, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.31560283687943264, + "acc_stderr": 0.027724989449509317, + "brier_score": 0.768002782091526, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_college_physics": { - "acc": 0.24509803921568626, - "acc_stderr": 0.04280105837364395, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.22549019607843138, + "acc_stderr": 0.041583075330832865, + "brier_score": 0.7977170964249326, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_high_school_physics": { - "acc": 0.33774834437086093, - "acc_stderr": 0.03861557546255168, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.2980132450331126, + "acc_stderr": 0.037345356767871984, + "brier_score": 0.7633345278522208, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_high_school_biology": { - "acc": 0.3870967741935484, - "acc_stderr": 0.02770935967503249, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.35161290322580646, + "acc_stderr": 0.027162537826948458, + "brier_score": 0.7338681018242579, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_college_biology": { - "acc": 0.3263888888888889, - "acc_stderr": 0.03921067198982266, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.3194444444444444, + "acc_stderr": 0.038990736873573344, + "brier_score": 0.7382610461426873, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_anatomy": { - "acc": 0.28888888888888886, - "acc_stderr": 0.0391545063041425, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.32592592592592595, + "acc_stderr": 0.040491220417025055, + "brier_score": 0.7194668604811008, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_college_chemistry": { - "acc": 0.36, - "acc_stderr": 0.048241815132442176, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.26, + "acc_stderr": 0.04408440022768079, + "brier_score": 0.7800486692301324, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_computer_security": { - "acc": 0.37, - "acc_stderr": 0.04852365870939099, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.46, + "acc_stderr": 0.05009082659620333, + "brier_score": 0.6713774316992404, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_college_computer_science": { - "acc": 0.37, - "acc_stderr": 0.04852365870939099, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.29, + "acc_stderr": 0.045604802157206845, + "brier_score": 0.8094644862958944, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_astronomy": { - "acc": 0.3157894736842105, - "acc_stderr": 0.037827289808654685, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.3355263157894737, + "acc_stderr": 0.03842498559395268, + "brier_score": 0.7766442411477616, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_college_mathematics": { - "acc": 0.32, - "acc_stderr": 0.04688261722621504, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.29, + "acc_stderr": 0.04560480215720684, + "brier_score": 0.7854849117108706, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_conceptual_physics": { - "acc": 0.34893617021276596, - "acc_stderr": 0.031158522131357783, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.3021276595744681, + "acc_stderr": 0.030017554471880557, + "brier_score": 0.777047512305051, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_abstract_algebra": { - "acc": 0.26, - "acc_stderr": 0.04408440022768078, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.27, + "acc_stderr": 0.04461960433384741, + "brier_score": 0.8814857314331535, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_high_school_computer_science": { - "acc": 0.34, - "acc_stderr": 0.04760952285695235, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "brier_score": 0.736066824294179, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_machine_learning": { - "acc": 0.33035714285714285, - "acc_stderr": 0.04464285714285714, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.35714285714285715, + "acc_stderr": 0.04547960999764376, + "brier_score": 0.779959477294934, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_high_school_chemistry": { - "acc": 0.2857142857142857, - "acc_stderr": 0.0317852971064275, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.2660098522167488, + "acc_stderr": 0.03108982600293753, + "brier_score": 0.7725297810535339, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_high_school_statistics": { - "acc": 0.375, - "acc_stderr": 0.033016908987210894, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.2962962962962963, + "acc_stderr": 0.031141447823536023, + "brier_score": 0.7689269329033727, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_elementary_mathematics": { - "acc": 0.2777777777777778, - "acc_stderr": 0.02306818884826111, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.2857142857142857, + "acc_stderr": 0.023266512213730564, + "brier_score": 0.8022422868333021, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_electrical_engineering": { "acc": 0.35172413793103446, - "acc_stderr": 0.03979236637497412, - "timestamp": "2024-11-07T02-15-50.682989" + "acc_stderr": 0.0397923663749741, + "brier_score": 0.7397187719586187, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "mmlu_high_school_mathematics": { - "acc": 0.26296296296296295, - "acc_stderr": 0.026842057873833713, - "timestamp": "2024-11-07T02-15-50.682989" + "acc": 0.22592592592592592, + "acc_stderr": 0.025497532639609553, + "brier_score": 0.8140482353582547, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T17-39-47.488962" }, "arc_challenge": { "acc": 0.44795221843003413, diff --git a/metadata/model_scores/results_mosaicml__mpt-7b-storywriter.json b/metadata/model_scores/results_mosaicml__mpt-7b-storywriter.json index 9b73ac3..b0caa1c 100644 --- a/metadata/model_scores/results_mosaicml__mpt-7b-storywriter.json +++ b/metadata/model_scores/results_mosaicml__mpt-7b-storywriter.json @@ -1,6 +1,6 @@ { "model_name": "mosaicml/mpt-7b-storywriter", - "last_updated": "2024-12-04 11:23:14.200213", + "last_updated": "2024-12-19 13:38:31.178657", "results": { "harness": { "minerva_math_precalc": { @@ -228,289 +228,403 @@ "timestamp": "2024-06-09T18-43-59.272112" }, "mmlu_world_religions": { - "acc": 0.23976608187134502, - "acc_stderr": 0.03274485211946956, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.26900584795321636, + "acc_stderr": 0.03401052620104089, + "brier_score": 0.753680202481827, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_formal_logic": { "acc": 0.21428571428571427, - "acc_stderr": 0.03670066451047181, - "timestamp": "2024-11-24T14-31-05.387078" + "acc_stderr": 0.03670066451047182, + "brier_score": 0.8038510938196719, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_prehistory": { - "acc": 0.26851851851851855, - "acc_stderr": 0.024659685185967294, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.25925925925925924, + "acc_stderr": 0.024383665531035454, + "brier_score": 0.7429447897247751, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_moral_scenarios": { - "acc": 0.24916201117318434, - "acc_stderr": 0.014465893829859923, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.2424581005586592, + "acc_stderr": 0.014333522059217887, + "brier_score": 0.7936312220620956, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_high_school_world_history": { - "acc": 0.3333333333333333, - "acc_stderr": 0.03068582059661081, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.23628691983122363, + "acc_stderr": 0.02765215314415925, + "brier_score": 0.7641102367810089, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_moral_disputes": { - "acc": 0.29190751445086704, - "acc_stderr": 0.024476994076247323, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.26878612716763006, + "acc_stderr": 0.023868003262500114, + "brier_score": 0.7481473957025903, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_professional_law": { - "acc": 0.25684485006518903, - "acc_stderr": 0.011158455853098862, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.2757496740547588, + "acc_stderr": 0.01141381360916099, + "brier_score": 0.7599821212753395, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_logical_fallacies": { - "acc": 0.26993865030674846, - "acc_stderr": 0.034878251684978906, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.26380368098159507, + "acc_stderr": 0.034624199316156234, + "brier_score": 0.7489141116387855, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_high_school_us_history": { - "acc": 0.3480392156862745, - "acc_stderr": 0.03343311240488418, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.25980392156862747, + "acc_stderr": 0.03077855467869326, + "brier_score": 0.758159885868019, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_philosophy": { - "acc": 0.26366559485530544, - "acc_stderr": 0.02502553850053234, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.3183279742765273, + "acc_stderr": 0.026457225067811032, + "brier_score": 0.7371383091149315, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_jurisprudence": { - "acc": 0.25925925925925924, - "acc_stderr": 0.04236511258094632, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.3148148148148148, + "acc_stderr": 0.04489931073591311, + "brier_score": 0.740167746960495, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_international_law": { - "acc": 0.24793388429752067, - "acc_stderr": 0.03941897526516303, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.34710743801652894, + "acc_stderr": 0.04345724570292535, + "brier_score": 0.7215996248957431, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_high_school_european_history": { - "acc": 0.24848484848484848, - "acc_stderr": 0.03374402644139405, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.30303030303030304, + "acc_stderr": 0.03588624800091709, + "brier_score": 0.7488523500000018, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_high_school_government_and_politics": { - "acc": 0.2694300518134715, - "acc_stderr": 0.03201867122877794, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.26424870466321243, + "acc_stderr": 0.03182155050916649, + "brier_score": 0.761883170110334, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_high_school_microeconomics": { - "acc": 0.23949579831932774, - "acc_stderr": 0.027722065493361266, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.21428571428571427, + "acc_stderr": 0.02665353159671549, + "brier_score": 0.768040669440023, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_high_school_geography": { - "acc": 0.2474747474747475, - "acc_stderr": 0.030746300742124522, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.20202020202020202, + "acc_stderr": 0.028606204289229872, + "brier_score": 0.776820768579571, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_high_school_psychology": { - "acc": 0.23669724770642203, - "acc_stderr": 0.01822407811729907, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.22935779816513763, + "acc_stderr": 0.018025349724618684, + "brier_score": 0.7668421972927677, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_public_relations": { - "acc": 0.38181818181818183, - "acc_stderr": 0.046534298079135075, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.3, + "acc_stderr": 0.04389311454644287, + "brier_score": 0.7466252547843161, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_us_foreign_policy": { - "acc": 0.28, - "acc_stderr": 0.04512608598542127, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 0.7420782541244603, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_sociology": { - "acc": 0.2835820895522388, - "acc_stderr": 0.031871875379197966, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.30845771144278605, + "acc_stderr": 0.032658195885126966, + "brier_score": 0.7544591020949951, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_high_school_macroeconomics": { - "acc": 0.25384615384615383, - "acc_stderr": 0.022066054378726257, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.2358974358974359, + "acc_stderr": 0.021525965407408726, + "brier_score": 0.7728504872846409, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_security_studies": { - "acc": 0.2816326530612245, - "acc_stderr": 0.028795185574291282, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.2163265306122449, + "acc_stderr": 0.026358916334904035, + "brier_score": 0.7685743597613134, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_professional_psychology": { - "acc": 0.27450980392156865, - "acc_stderr": 0.0180540274588152, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.2549019607843137, + "acc_stderr": 0.017630827375148383, + "brier_score": 0.754200107568594, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_human_sexuality": { - "acc": 0.2748091603053435, - "acc_stderr": 0.03915345408847836, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.22900763358778625, + "acc_stderr": 0.036853466317118506, + "brier_score": 0.7672084282956686, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_econometrics": { - "acc": 0.2631578947368421, - "acc_stderr": 0.04142439719489359, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.24561403508771928, + "acc_stderr": 0.04049339297748142, + "brier_score": 0.7884101694016207, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_miscellaneous": { - "acc": 0.30395913154533843, - "acc_stderr": 0.016448321686769043, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.3243933588761175, + "acc_stderr": 0.016740929047162702, + "brier_score": 0.7347917320598347, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_marketing": { - "acc": 0.29914529914529914, - "acc_stderr": 0.029996951858349476, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.2692307692307692, + "acc_stderr": 0.02905858830374884, + "brier_score": 0.7496901398504675, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_management": { - "acc": 0.24271844660194175, - "acc_stderr": 0.04245022486384495, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.2524271844660194, + "acc_stderr": 0.04301250399690878, + "brier_score": 0.7663697778928393, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_nutrition": { - "acc": 0.30392156862745096, - "acc_stderr": 0.026336613469046626, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.238562091503268, + "acc_stderr": 0.024404394928087873, + "brier_score": 0.772850127762166, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_medical_genetics": { - "acc": 0.26, - "acc_stderr": 0.0440844002276808, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.23, + "acc_stderr": 0.042295258468165065, + "brier_score": 0.768684437705436, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_human_aging": { - "acc": 0.32286995515695066, - "acc_stderr": 0.03138147637575498, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.3542600896860987, + "acc_stderr": 0.032100621541349864, + "brier_score": 0.7316352696315931, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_professional_medicine": { - "acc": 0.21691176470588236, - "acc_stderr": 0.025035845227711264, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.19852941176470587, + "acc_stderr": 0.024231013370541076, + "brier_score": 0.7894777255106187, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_college_medicine": { - "acc": 0.23699421965317918, - "acc_stderr": 0.03242414757483098, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.2832369942196532, + "acc_stderr": 0.03435568056047874, + "brier_score": 0.7649221885589982, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_business_ethics": { - "acc": 0.28, - "acc_stderr": 0.045126085985421296, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.33, + "acc_stderr": 0.047258156262526045, + "brier_score": 0.7479446150506702, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_clinical_knowledge": { "acc": 0.26037735849056604, "acc_stderr": 0.0270087660907081, - "timestamp": "2024-11-24T14-31-05.387078" + "brier_score": 0.7615723376439071, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_global_facts": { - "acc": 0.27, - "acc_stderr": 0.044619604333847394, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.39, + "acc_stderr": 0.04902071300001974, + "brier_score": 0.7120079089893042, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_virology": { - "acc": 0.3373493975903614, - "acc_stderr": 0.03680783690727581, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.3313253012048193, + "acc_stderr": 0.03664314777288086, + "brier_score": 0.7541565314631925, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_professional_accounting": { - "acc": 0.2801418439716312, - "acc_stderr": 0.02678917235114025, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.2907801418439716, + "acc_stderr": 0.027090664368353178, + "brier_score": 0.7545586416754995, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_college_physics": { - "acc": 0.20588235294117646, - "acc_stderr": 0.04023382273617747, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.2647058823529412, + "acc_stderr": 0.0438986995680878, + "brier_score": 0.7868482264949791, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_high_school_physics": { - "acc": 0.2582781456953642, - "acc_stderr": 0.035737053147634576, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.23841059602649006, + "acc_stderr": 0.03479185572599661, + "brier_score": 0.7700271734827929, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_high_school_biology": { - "acc": 0.25483870967741934, - "acc_stderr": 0.024790118459332204, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.3032258064516129, + "acc_stderr": 0.026148685930671742, + "brier_score": 0.7505801621274838, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_college_biology": { - "acc": 0.2569444444444444, - "acc_stderr": 0.03653946969442099, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.25, + "acc_stderr": 0.03621034121889507, + "brier_score": 0.7704275393486419, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_anatomy": { - "acc": 0.2814814814814815, - "acc_stderr": 0.038850042458002554, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.28888888888888886, + "acc_stderr": 0.0391545063041425, + "brier_score": 0.7486436018611725, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_college_chemistry": { - "acc": 0.18, - "acc_stderr": 0.03861229196653695, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.19, + "acc_stderr": 0.03942772444036623, + "brier_score": 0.8148346665464601, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_computer_security": { - "acc": 0.31, - "acc_stderr": 0.04648231987117316, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.33, + "acc_stderr": 0.047258156262526045, + "brier_score": 0.7270989693573726, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_college_computer_science": { - "acc": 0.23, - "acc_stderr": 0.04229525846816506, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.12, + "acc_stderr": 0.03265986323710906, + "brier_score": 0.8072218865613245, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_astronomy": { - "acc": 0.2236842105263158, - "acc_stderr": 0.033911609343436025, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.26973684210526316, + "acc_stderr": 0.03611780560284898, + "brier_score": 0.7628450006384498, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_college_mathematics": { - "acc": 0.27, - "acc_stderr": 0.044619604333847415, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.22, + "acc_stderr": 0.04163331998932269, + "brier_score": 0.7757384831926484, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_conceptual_physics": { - "acc": 0.3191489361702128, - "acc_stderr": 0.03047297336338004, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.32340425531914896, + "acc_stderr": 0.03057944277361035, + "brier_score": 0.7454022711302398, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_abstract_algebra": { - "acc": 0.27, - "acc_stderr": 0.04461960433384741, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "brier_score": 0.7519858148417691, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_high_school_computer_science": { - "acc": 0.26, - "acc_stderr": 0.04408440022768077, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.27, + "acc_stderr": 0.044619604333847394, + "brier_score": 0.7488524750876776, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_machine_learning": { - "acc": 0.25892857142857145, - "acc_stderr": 0.04157751539865629, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.2767857142857143, + "acc_stderr": 0.04246624336697624, + "brier_score": 0.7810329226775689, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_high_school_chemistry": { - "acc": 0.24630541871921183, - "acc_stderr": 0.03031509928561773, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.2561576354679803, + "acc_stderr": 0.0307127300709826, + "brier_score": 0.7607518115873994, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_high_school_statistics": { - "acc": 0.2916666666666667, - "acc_stderr": 0.030998666304560534, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.1712962962962963, + "acc_stderr": 0.025695341643824685, + "brier_score": 0.800553431750079, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_elementary_mathematics": { - "acc": 0.2751322751322751, - "acc_stderr": 0.023000086859068642, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.2619047619047619, + "acc_stderr": 0.022644212615525218, + "brier_score": 0.7751121777807123, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_electrical_engineering": { - "acc": 0.25517241379310346, - "acc_stderr": 0.03632984052707842, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.2896551724137931, + "acc_stderr": 0.03780019230438014, + "brier_score": 0.7521340858384736, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "mmlu_high_school_mathematics": { - "acc": 0.25925925925925924, - "acc_stderr": 0.02671924078371216, - "timestamp": "2024-11-24T14-31-05.387078" + "acc": 0.25555555555555554, + "acc_stderr": 0.02659393910184407, + "brier_score": 0.775342769692641, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-10-46.853781" }, "arc_challenge": { "acc": 0.4274744027303754, diff --git a/metadata/model_scores/results_mosaicml__mpt-7b.json b/metadata/model_scores/results_mosaicml__mpt-7b.json index f255eaf..f658a2f 100644 --- a/metadata/model_scores/results_mosaicml__mpt-7b.json +++ b/metadata/model_scores/results_mosaicml__mpt-7b.json @@ -1,6 +1,6 @@ { "model_name": "mosaicml/mpt-7b", - "last_updated": "2024-12-04 11:24:06.088694", + "last_updated": "2024-12-19 13:39:32.490198", "results": { "harness": { "minerva_math_precalc": { @@ -228,289 +228,403 @@ "timestamp": "2024-06-09T16-58-32.555159" }, "mmlu_world_religions": { - "acc": 0.3333333333333333, - "acc_stderr": 0.03615507630310937, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.3157894736842105, + "acc_stderr": 0.03565079670708311, + "brier_score": 0.728741684015239, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_formal_logic": { - "acc": 0.19047619047619047, - "acc_stderr": 0.03512207412302053, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.2777777777777778, + "acc_stderr": 0.04006168083848879, + "brier_score": 0.7523749874783903, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_prehistory": { - "acc": 0.3117283950617284, - "acc_stderr": 0.02577311116963046, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.3148148148148148, + "acc_stderr": 0.025842248700902168, + "brier_score": 0.7309064722060028, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_moral_scenarios": { - "acc": 0.25139664804469275, - "acc_stderr": 0.014508979453553991, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.23687150837988827, + "acc_stderr": 0.014219570788103982, + "brier_score": 0.7937717222558787, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_high_school_world_history": { - "acc": 0.270042194092827, - "acc_stderr": 0.028900721906293426, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.3080168776371308, + "acc_stderr": 0.030052389335605702, + "brier_score": 0.7382162992563585, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_moral_disputes": { - "acc": 0.2861271676300578, - "acc_stderr": 0.024332146779134128, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.3092485549132948, + "acc_stderr": 0.024883140570071755, + "brier_score": 0.7411018298260529, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_professional_law": { - "acc": 0.2633637548891786, - "acc_stderr": 0.011249506403605284, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.2666232073011734, + "acc_stderr": 0.011293836031612143, + "brier_score": 0.7487617393696027, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_logical_fallacies": { - "acc": 0.2331288343558282, - "acc_stderr": 0.03322015795776741, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.26993865030674846, + "acc_stderr": 0.03487825168497892, + "brier_score": 0.7464647521267894, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_high_school_us_history": { - "acc": 0.2696078431372549, - "acc_stderr": 0.03114557065948678, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.29411764705882354, + "acc_stderr": 0.031980016601150706, + "brier_score": 0.7394602455358426, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_philosophy": { "acc": 0.3086816720257235, "acc_stderr": 0.026236965881153266, - "timestamp": "2024-11-19T21-35-21.939819" + "brier_score": 0.7320864499183289, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_jurisprudence": { - "acc": 0.3425925925925926, - "acc_stderr": 0.045879047413018105, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.3611111111111111, + "acc_stderr": 0.04643454608906275, + "brier_score": 0.7192370502154816, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_international_law": { - "acc": 0.23140495867768596, - "acc_stderr": 0.03849856098794089, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.4462809917355372, + "acc_stderr": 0.0453793517794788, + "brier_score": 0.7043917930738771, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_high_school_european_history": { - "acc": 0.24242424242424243, - "acc_stderr": 0.03346409881055953, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.3090909090909091, + "acc_stderr": 0.03608541011573967, + "brier_score": 0.7349772415005317, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_high_school_government_and_politics": { - "acc": 0.31088082901554404, - "acc_stderr": 0.03340361906276588, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.3160621761658031, + "acc_stderr": 0.033553973696861736, + "brier_score": 0.7290051141753543, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_high_school_microeconomics": { - "acc": 0.2689075630252101, - "acc_stderr": 0.02880139219363127, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.3319327731092437, + "acc_stderr": 0.030588697013783663, + "brier_score": 0.7427459476388498, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_high_school_geography": { - "acc": 0.22727272727272727, - "acc_stderr": 0.02985751567338641, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.2828282828282828, + "acc_stderr": 0.03208779558786751, + "brier_score": 0.7517428390896269, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_high_school_psychology": { - "acc": 0.26238532110091745, - "acc_stderr": 0.018861885021534745, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.27339449541284405, + "acc_stderr": 0.01910929984609828, + "brier_score": 0.7492081397364514, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_public_relations": { - "acc": 0.3090909090909091, - "acc_stderr": 0.044262946482000985, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.3181818181818182, + "acc_stderr": 0.04461272175910507, + "brier_score": 0.7322864162960925, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_us_foreign_policy": { - "acc": 0.46, - "acc_stderr": 0.05009082659620333, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "brier_score": 0.7209992823667533, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_sociology": { - "acc": 0.20398009950248755, - "acc_stderr": 0.02849317624532607, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.3283582089552239, + "acc_stderr": 0.033206858897443244, + "brier_score": 0.7210230646478174, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_high_school_macroeconomics": { - "acc": 0.2743589743589744, - "acc_stderr": 0.0226227657674932, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.24871794871794872, + "acc_stderr": 0.0219169577092138, + "brier_score": 0.7482215522041382, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_security_studies": { - "acc": 0.3224489795918367, - "acc_stderr": 0.029923100563683906, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.2653061224489796, + "acc_stderr": 0.028263889943784606, + "brier_score": 0.7401749633097108, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_professional_psychology": { - "acc": 0.2679738562091503, - "acc_stderr": 0.017917974069594726, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.3202614379084967, + "acc_stderr": 0.018875682938069443, + "brier_score": 0.7367828947482863, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_human_sexuality": { - "acc": 0.3053435114503817, - "acc_stderr": 0.040393149787245605, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.31297709923664124, + "acc_stderr": 0.04066962905677698, + "brier_score": 0.7360726453126495, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_econometrics": { - "acc": 0.2894736842105263, - "acc_stderr": 0.04266339443159394, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.2543859649122807, + "acc_stderr": 0.040969851398436716, + "brier_score": 0.7811233293785654, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_miscellaneous": { - "acc": 0.29757343550446996, - "acc_stderr": 0.01634911191290942, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.37037037037037035, + "acc_stderr": 0.017268607560005776, + "brier_score": 0.7089064656141576, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_marketing": { - "acc": 0.3333333333333333, - "acc_stderr": 0.030882736974138653, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.29914529914529914, + "acc_stderr": 0.029996951858349483, + "brier_score": 0.7226735305047545, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_management": { - "acc": 0.2621359223300971, - "acc_stderr": 0.04354631077260595, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.32038834951456313, + "acc_stderr": 0.04620284082280039, + "brier_score": 0.7464823125999717, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_nutrition": { - "acc": 0.27124183006535946, - "acc_stderr": 0.025457756696667878, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.2549019607843137, + "acc_stderr": 0.02495418432487991, + "brier_score": 0.748069712172246, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_medical_genetics": { - "acc": 0.35, - "acc_stderr": 0.0479372485441102, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.33, + "acc_stderr": 0.04725815626252605, + "brier_score": 0.7375088711867108, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_human_aging": { - "acc": 0.3542600896860987, - "acc_stderr": 0.032100621541349864, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.39461883408071746, + "acc_stderr": 0.03280400504755291, + "brier_score": 0.7173562027693366, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_professional_medicine": { - "acc": 0.20588235294117646, - "acc_stderr": 0.024562204314142314, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.23529411764705882, + "acc_stderr": 0.025767252010855956, + "brier_score": 0.7525455665900059, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_college_medicine": { - "acc": 0.24277456647398843, - "acc_stderr": 0.0326926380614177, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.27167630057803466, + "acc_stderr": 0.03391750322321659, + "brier_score": 0.7542974573066814, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_business_ethics": { - "acc": 0.28, - "acc_stderr": 0.04512608598542127, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.4, + "acc_stderr": 0.04923659639173309, + "brier_score": 0.7250757977146242, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_clinical_knowledge": { - "acc": 0.25660377358490566, - "acc_stderr": 0.02688064788905199, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.32452830188679244, + "acc_stderr": 0.028815615713432115, + "brier_score": 0.746111321222342, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_global_facts": { - "acc": 0.22, - "acc_stderr": 0.04163331998932269, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.39, + "acc_stderr": 0.04902071300001974, + "brier_score": 0.7102197513122844, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_virology": { - "acc": 0.3132530120481928, - "acc_stderr": 0.036108050180310235, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.3192771084337349, + "acc_stderr": 0.03629335329947859, + "brier_score": 0.7342408644320478, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_professional_accounting": { - "acc": 0.25177304964539005, - "acc_stderr": 0.0258921511567094, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.2907801418439716, + "acc_stderr": 0.027090664368353178, + "brier_score": 0.7492878539459001, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_college_physics": { - "acc": 0.20588235294117646, - "acc_stderr": 0.04023382273617749, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.27450980392156865, + "acc_stderr": 0.044405219061793254, + "brier_score": 0.7787676832484953, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_high_school_physics": { - "acc": 0.25165562913907286, - "acc_stderr": 0.03543304234389985, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.2052980132450331, + "acc_stderr": 0.03297986648473835, + "brier_score": 0.7554343222342074, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_high_school_biology": { - "acc": 0.25806451612903225, - "acc_stderr": 0.02489246917246283, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.3, + "acc_stderr": 0.026069362295335127, + "brier_score": 0.7446064679583299, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_college_biology": { - "acc": 0.2986111111111111, - "acc_stderr": 0.03827052357950756, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.2916666666666667, + "acc_stderr": 0.038009680605548594, + "brier_score": 0.7416430063536041, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_anatomy": { - "acc": 0.25925925925925924, - "acc_stderr": 0.03785714465066654, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.28888888888888886, + "acc_stderr": 0.03915450630414251, + "brier_score": 0.7290725108490472, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_college_chemistry": { - "acc": 0.27, - "acc_stderr": 0.0446196043338474, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.21, + "acc_stderr": 0.04093601807403326, + "brier_score": 0.781694105501675, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_computer_security": { - "acc": 0.27, - "acc_stderr": 0.044619604333847394, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.39, + "acc_stderr": 0.04902071300001975, + "brier_score": 0.7111190458395429, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_college_computer_science": { - "acc": 0.35, - "acc_stderr": 0.0479372485441102, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.15, + "acc_stderr": 0.0358870281282637, + "brier_score": 0.7767318512632042, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_astronomy": { - "acc": 0.26973684210526316, - "acc_stderr": 0.036117805602848975, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.2565789473684211, + "acc_stderr": 0.0355418036802569, + "brier_score": 0.757864750388492, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_college_mathematics": { - "acc": 0.35, - "acc_stderr": 0.047937248544110196, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.24, + "acc_stderr": 0.042923469599092816, + "brier_score": 0.7578289940819812, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_conceptual_physics": { "acc": 0.32340425531914896, - "acc_stderr": 0.03057944277361034, - "timestamp": "2024-11-19T21-35-21.939819" + "acc_stderr": 0.030579442773610344, + "brier_score": 0.7403692614539054, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_abstract_algebra": { - "acc": 0.19, - "acc_stderr": 0.039427724440366234, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.29, + "acc_stderr": 0.045604802157206845, + "brier_score": 0.7513816282514579, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_high_school_computer_science": { "acc": 0.23, - "acc_stderr": 0.042295258468165044, - "timestamp": "2024-11-19T21-35-21.939819" + "acc_stderr": 0.04229525846816506, + "brier_score": 0.7392790076338863, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_machine_learning": { - "acc": 0.35714285714285715, - "acc_stderr": 0.04547960999764376, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.29464285714285715, + "acc_stderr": 0.0432704093257873, + "brier_score": 0.74507395792578, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_high_school_chemistry": { - "acc": 0.16748768472906403, - "acc_stderr": 0.026273086047535425, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.2512315270935961, + "acc_stderr": 0.030516530732694436, + "brier_score": 0.7485835684629456, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_high_school_statistics": { - "acc": 0.27314814814814814, - "acc_stderr": 0.03038805130167812, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.21296296296296297, + "acc_stderr": 0.027920963147993666, + "brier_score": 0.7700312540477746, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_elementary_mathematics": { - "acc": 0.24867724867724866, - "acc_stderr": 0.022261817692400158, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.2830687830687831, + "acc_stderr": 0.023201392938194974, + "brier_score": 0.7648506953710482, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_electrical_engineering": { - "acc": 0.3448275862068966, - "acc_stderr": 0.03960933549451208, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.2827586206896552, + "acc_stderr": 0.03752833958003336, + "brier_score": 0.7377471340632694, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "mmlu_high_school_mathematics": { - "acc": 0.29259259259259257, - "acc_stderr": 0.027738969632176088, - "timestamp": "2024-11-19T21-35-21.939819" + "acc": 0.2222222222222222, + "acc_stderr": 0.025348097468097852, + "brier_score": 0.7762464169473738, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T18-49-04.168570" }, "arc_challenge": { "acc": 0.4351535836177474, diff --git a/metadata/model_scores/results_openlm-research__open_llama_3b_v2.json b/metadata/model_scores/results_openlm-research__open_llama_3b_v2.json index b337e47..0da86cc 100644 --- a/metadata/model_scores/results_openlm-research__open_llama_3b_v2.json +++ b/metadata/model_scores/results_openlm-research__open_llama_3b_v2.json @@ -1,6 +1,6 @@ { "model_name": "openlm-research/open_llama_3b_v2", - "last_updated": "2024-12-04 11:25:40.016223", + "last_updated": "2024-12-19 13:41:53.442327", "results": { "harness": { "minerva_math_precalc": { @@ -228,289 +228,403 @@ "timestamp": "2024-06-13T11-35-25.155228" }, "mmlu_world_religions": { - "acc": 0.3157894736842105, - "acc_stderr": 0.03565079670708312, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.2573099415204678, + "acc_stderr": 0.03352799844161865, + "brier_score": 0.7480009231732265, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_formal_logic": { - "acc": 0.23809523809523808, - "acc_stderr": 0.03809523809523809, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.21428571428571427, + "acc_stderr": 0.03670066451047182, + "brier_score": 0.7615843108140404, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_prehistory": { - "acc": 0.2839506172839506, - "acc_stderr": 0.02508947852376513, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.2623456790123457, + "acc_stderr": 0.02447722285613511, + "brier_score": 0.7461830940453977, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_moral_scenarios": { - "acc": 0.2435754189944134, - "acc_stderr": 0.01435591196476786, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.2569832402234637, + "acc_stderr": 0.01461446582196633, + "brier_score": 0.7634191585528008, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_high_school_world_history": { - "acc": 0.2489451476793249, - "acc_stderr": 0.028146970599422644, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.27848101265822783, + "acc_stderr": 0.02917868230484255, + "brier_score": 0.7547953289576513, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_moral_disputes": { - "acc": 0.26011560693641617, - "acc_stderr": 0.023618678310069374, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.26878612716763006, + "acc_stderr": 0.023868003262500125, + "brier_score": 0.7489207949995742, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_professional_law": { - "acc": 0.2470664928292047, - "acc_stderr": 0.01101575225527935, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.25097783572359844, + "acc_stderr": 0.011073730299187215, + "brier_score": 0.759512116632909, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_logical_fallacies": { - "acc": 0.2085889570552147, - "acc_stderr": 0.031921934489347235, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.24539877300613497, + "acc_stderr": 0.03380939813943354, + "brier_score": 0.7544872650836869, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_high_school_us_history": { - "acc": 0.23039215686274508, - "acc_stderr": 0.029554292605695066, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.25, + "acc_stderr": 0.03039153369274154, + "brier_score": 0.7535941268466293, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_philosophy": { "acc": 0.27009646302250806, "acc_stderr": 0.025218040373410622, - "timestamp": "2024-11-06T18-40-41.914183" + "brier_score": 0.7493121131400111, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_jurisprudence": { "acc": 0.2962962962962963, - "acc_stderr": 0.04414343666854933, - "timestamp": "2024-11-06T18-40-41.914183" + "acc_stderr": 0.044143436668549335, + "brier_score": 0.7561458409644394, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_international_law": { - "acc": 0.2809917355371901, - "acc_stderr": 0.04103203830514511, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.2727272727272727, + "acc_stderr": 0.04065578140908705, + "brier_score": 0.7441216148127173, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_high_school_european_history": { - "acc": 0.24242424242424243, - "acc_stderr": 0.033464098810559534, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.21212121212121213, + "acc_stderr": 0.031922715695482974, + "brier_score": 0.7566455758380454, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_high_school_government_and_politics": { - "acc": 0.23316062176165803, - "acc_stderr": 0.030516111371476008, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.22797927461139897, + "acc_stderr": 0.030276909945178267, + "brier_score": 0.7633302138074438, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_high_school_microeconomics": { - "acc": 0.28991596638655465, - "acc_stderr": 0.029472485833136098, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.20588235294117646, + "acc_stderr": 0.026265024608275882, + "brier_score": 0.7594553429265166, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_high_school_geography": { - "acc": 0.21717171717171718, - "acc_stderr": 0.02937661648494563, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.2222222222222222, + "acc_stderr": 0.029620227874790486, + "brier_score": 0.7576363270233148, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_high_school_psychology": { "acc": 0.24036697247706423, "acc_stderr": 0.01832060732096407, - "timestamp": "2024-11-06T18-40-41.914183" + "brier_score": 0.7565590314930034, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_public_relations": { - "acc": 0.3181818181818182, - "acc_stderr": 0.04461272175910507, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.2727272727272727, + "acc_stderr": 0.04265792110940588, + "brier_score": 0.7469096593763332, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_us_foreign_policy": { - "acc": 0.34, - "acc_stderr": 0.04760952285695235, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.24, + "acc_stderr": 0.04292346959909284, + "brier_score": 0.7470080039570334, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_sociology": { - "acc": 0.25870646766169153, - "acc_stderr": 0.030965903123573033, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.2537313432835821, + "acc_stderr": 0.030769444967296007, + "brier_score": 0.7548193420044138, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_high_school_macroeconomics": { - "acc": 0.24871794871794872, - "acc_stderr": 0.021916957709213803, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.23333333333333334, + "acc_stderr": 0.021444547301560472, + "brier_score": 0.7595149623585433, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_security_studies": { - "acc": 0.3224489795918367, - "acc_stderr": 0.02992310056368391, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.22040816326530613, + "acc_stderr": 0.026537045312145298, + "brier_score": 0.7512240644391838, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_professional_psychology": { - "acc": 0.27124183006535946, - "acc_stderr": 0.017986615304030316, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.26143790849673204, + "acc_stderr": 0.017776947157528044, + "brier_score": 0.7506030256061083, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_human_sexuality": { - "acc": 0.24427480916030533, - "acc_stderr": 0.03768335959728742, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.20610687022900764, + "acc_stderr": 0.03547771004159463, + "brier_score": 0.7542686799934393, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_econometrics": { - "acc": 0.24561403508771928, - "acc_stderr": 0.04049339297748142, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.21052631578947367, + "acc_stderr": 0.038351539543994194, + "brier_score": 0.7614829132464986, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_miscellaneous": { - "acc": 0.2835249042145594, - "acc_stderr": 0.016117318166832276, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.29246487867177523, + "acc_stderr": 0.016267000684598652, + "brier_score": 0.7479227764357439, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_marketing": { - "acc": 0.29914529914529914, - "acc_stderr": 0.02999695185834948, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.2948717948717949, + "acc_stderr": 0.02987257770889117, + "brier_score": 0.7485007588679611, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_management": { - "acc": 0.2524271844660194, - "acc_stderr": 0.04301250399690878, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.23300970873786409, + "acc_stderr": 0.041858325989283164, + "brier_score": 0.76271521896551, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_nutrition": { - "acc": 0.26143790849673204, - "acc_stderr": 0.025160998214292456, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.24836601307189543, + "acc_stderr": 0.02473998135511359, + "brier_score": 0.7542974540466784, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_medical_genetics": { - "acc": 0.22, - "acc_stderr": 0.04163331998932269, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "brier_score": 0.735961193100805, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_human_aging": { - "acc": 0.3811659192825112, - "acc_stderr": 0.03259625118416827, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.336322869955157, + "acc_stderr": 0.031708824268455005, + "brier_score": 0.7337681857323938, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_professional_medicine": { - "acc": 0.21691176470588236, - "acc_stderr": 0.025035845227711274, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.21323529411764705, + "acc_stderr": 0.02488097151229425, + "brier_score": 0.779047316094294, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_college_medicine": { - "acc": 0.2254335260115607, - "acc_stderr": 0.03186209851641143, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.23699421965317918, + "acc_stderr": 0.03242414757483098, + "brier_score": 0.7587275117336465, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_business_ethics": { - "acc": 0.41, - "acc_stderr": 0.049431107042371025, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.28, + "acc_stderr": 0.04512608598542127, + "brier_score": 0.7355173081904621, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_clinical_knowledge": { - "acc": 0.29056603773584905, - "acc_stderr": 0.027943219989337145, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.2339622641509434, + "acc_stderr": 0.02605529690115292, + "brier_score": 0.7554649912770037, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_global_facts": { - "acc": 0.31, - "acc_stderr": 0.04648231987117316, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.35, + "acc_stderr": 0.047937248544110196, + "brier_score": 0.7279422265069059, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_virology": { - "acc": 0.3373493975903614, - "acc_stderr": 0.0368078369072758, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.30120481927710846, + "acc_stderr": 0.03571609230053481, + "brier_score": 0.7404351214968269, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_professional_accounting": { - "acc": 0.30141843971631205, - "acc_stderr": 0.02737412888263115, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.24822695035460993, + "acc_stderr": 0.025770015644290406, + "brier_score": 0.7517030850500305, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_college_physics": { - "acc": 0.21568627450980393, - "acc_stderr": 0.04092563958237654, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.20588235294117646, + "acc_stderr": 0.04023382273617747, + "brier_score": 0.7568115594569838, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_high_school_physics": { - "acc": 0.32450331125827814, - "acc_stderr": 0.03822746937658753, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.31788079470198677, + "acc_stderr": 0.03802039760107903, + "brier_score": 0.7428996547009279, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_high_school_biology": { - "acc": 0.23548387096774193, - "acc_stderr": 0.024137632429337717, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.25806451612903225, + "acc_stderr": 0.02489246917246283, + "brier_score": 0.7504824526624697, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_college_biology": { - "acc": 0.24305555555555555, - "acc_stderr": 0.03586879280080341, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.20833333333333334, + "acc_stderr": 0.033961162058453336, + "brier_score": 0.7527306122742272, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_anatomy": { - "acc": 0.23703703703703705, - "acc_stderr": 0.03673731683969506, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.2740740740740741, + "acc_stderr": 0.03853254836552003, + "brier_score": 0.7443037185127753, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_college_chemistry": { - "acc": 0.24, - "acc_stderr": 0.04292346959909284, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.22, + "acc_stderr": 0.0416333199893227, + "brier_score": 0.760267604530027, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_computer_security": { - "acc": 0.32, - "acc_stderr": 0.04688261722621505, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "brier_score": 0.7429189011446976, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_college_computer_science": { - "acc": 0.23, - "acc_stderr": 0.04229525846816507, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.26, + "acc_stderr": 0.044084400227680794, + "brier_score": 0.7496145861335097, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_astronomy": { "acc": 0.26973684210526316, - "acc_stderr": 0.036117805602848975, - "timestamp": "2024-11-06T18-40-41.914183" + "acc_stderr": 0.03611780560284898, + "brier_score": 0.7488921848629451, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_college_mathematics": { - "acc": 0.32, - "acc_stderr": 0.046882617226215034, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.28, + "acc_stderr": 0.045126085985421296, + "brier_score": 0.7419170181348044, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_conceptual_physics": { - "acc": 0.3446808510638298, - "acc_stderr": 0.031068985963122145, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.2, + "acc_stderr": 0.026148818018424506, + "brier_score": 0.7562972788541444, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_abstract_algebra": { - "acc": 0.26, - "acc_stderr": 0.04408440022768079, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.24, + "acc_stderr": 0.042923469599092816, + "brier_score": 0.7608485266619526, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_high_school_computer_science": { - "acc": 0.26, - "acc_stderr": 0.0440844002276808, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.32, + "acc_stderr": 0.04688261722621505, + "brier_score": 0.7334407115286586, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_machine_learning": { - "acc": 0.29464285714285715, - "acc_stderr": 0.0432704093257873, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.23214285714285715, + "acc_stderr": 0.04007341809755805, + "brier_score": 0.7489247129769812, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_high_school_chemistry": { - "acc": 0.2413793103448276, - "acc_stderr": 0.030108330718011625, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.27586206896551724, + "acc_stderr": 0.03144712581678242, + "brier_score": 0.748090631862592, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_high_school_statistics": { - "acc": 0.19444444444444445, - "acc_stderr": 0.026991454502036726, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.1712962962962963, + "acc_stderr": 0.02569534164382468, + "brier_score": 0.7684251083451906, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_elementary_mathematics": { - "acc": 0.29365079365079366, - "acc_stderr": 0.02345603738398203, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.2619047619047619, + "acc_stderr": 0.022644212615525218, + "brier_score": 0.7539737149923048, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_electrical_engineering": { - "acc": 0.20689655172413793, - "acc_stderr": 0.03375672449560553, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.2896551724137931, + "acc_stderr": 0.03780019230438015, + "brier_score": 0.7435242337059804, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "mmlu_high_school_mathematics": { - "acc": 0.2222222222222222, - "acc_stderr": 0.025348097468097845, - "timestamp": "2024-11-06T18-40-41.914183" + "acc": 0.24814814814814815, + "acc_stderr": 0.026335739404055803, + "brier_score": 0.7540582461553792, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-17-55.916462" }, "arc_challenge": { "acc": 0.36006825938566556, diff --git a/metadata/model_scores/results_openlm-research__open_llama_7b.json b/metadata/model_scores/results_openlm-research__open_llama_7b.json index 4f55ebf..429639c 100644 --- a/metadata/model_scores/results_openlm-research__open_llama_7b.json +++ b/metadata/model_scores/results_openlm-research__open_llama_7b.json @@ -1,6 +1,6 @@ { "model_name": "openlm-research/open_llama_7b", - "last_updated": "2024-12-04 11:24:44.228667", + "last_updated": "2024-12-19 13:40:38.142595", "results": { "harness": { "minerva_math_precalc": { @@ -228,289 +228,403 @@ "timestamp": "2024-06-07T07-56-45.755035" }, "mmlu_world_religions": { - "acc": 0.3684210526315789, - "acc_stderr": 0.03699658017656878, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.38011695906432746, + "acc_stderr": 0.037229657413855394, + "brier_score": 0.7115781518397424, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_formal_logic": { - "acc": 0.24603174603174602, - "acc_stderr": 0.03852273364924316, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.23015873015873015, + "acc_stderr": 0.037649508797906066, + "brier_score": 0.7582250938895074, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_prehistory": { - "acc": 0.3117283950617284, - "acc_stderr": 0.02577311116963045, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.28703703703703703, + "acc_stderr": 0.02517104191530968, + "brier_score": 0.7433593871803341, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_moral_scenarios": { "acc": 0.24692737430167597, - "acc_stderr": 0.014422292204808838, - "timestamp": "2024-11-19T19-17-59.185280" + "acc_stderr": 0.01442229220480884, + "brier_score": 0.7733353691881525, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_high_school_world_history": { - "acc": 0.29535864978902954, - "acc_stderr": 0.02969633871342288, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.3206751054852321, + "acc_stderr": 0.030381931949990407, + "brier_score": 0.7400741865893943, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_moral_disputes": { - "acc": 0.3352601156069364, - "acc_stderr": 0.02541600377316555, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.3208092485549133, + "acc_stderr": 0.025131000233647886, + "brier_score": 0.7367835273826375, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_professional_law": { - "acc": 0.2627118644067797, - "acc_stderr": 0.01124054551499567, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.24967405475880053, + "acc_stderr": 0.011054538377832303, + "brier_score": 0.7541500718985059, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_logical_fallacies": { - "acc": 0.3006134969325153, - "acc_stderr": 0.03602511318806771, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.27607361963190186, + "acc_stderr": 0.0351238528370505, + "brier_score": 0.7540611090931192, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_high_school_us_history": { - "acc": 0.3235294117647059, - "acc_stderr": 0.03283472056108566, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.31862745098039214, + "acc_stderr": 0.032702871814820796, + "brier_score": 0.737559011238905, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_philosophy": { - "acc": 0.2829581993569132, - "acc_stderr": 0.025583062489984824, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.2958199356913183, + "acc_stderr": 0.025922371788818774, + "brier_score": 0.7431260488193858, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_jurisprudence": { - "acc": 0.3425925925925926, - "acc_stderr": 0.045879047413018126, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.3055555555555556, + "acc_stderr": 0.044531975073749834, + "brier_score": 0.7462423092896351, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_international_law": { - "acc": 0.39669421487603307, - "acc_stderr": 0.044658697805310094, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.3305785123966942, + "acc_stderr": 0.04294340845212094, + "brier_score": 0.7250984481128241, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_high_school_european_history": { - "acc": 0.296969696969697, - "acc_stderr": 0.03567969772268046, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.30303030303030304, + "acc_stderr": 0.03588624800091707, + "brier_score": 0.7400033896016402, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_high_school_government_and_politics": { - "acc": 0.35233160621761656, - "acc_stderr": 0.03447478286414358, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.35751295336787564, + "acc_stderr": 0.03458816042181004, + "brier_score": 0.7329061845026962, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_high_school_microeconomics": { - "acc": 0.3025210084033613, - "acc_stderr": 0.02983796238829193, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.29831932773109243, + "acc_stderr": 0.029719142876342856, + "brier_score": 0.7411659475083163, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_high_school_geography": { - "acc": 0.3484848484848485, - "acc_stderr": 0.033948539651564025, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.3333333333333333, + "acc_stderr": 0.03358618145732524, + "brier_score": 0.7393613997273552, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_high_school_psychology": { - "acc": 0.3577981651376147, - "acc_stderr": 0.020552060784827818, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.25137614678899084, + "acc_stderr": 0.01859920636028741, + "brier_score": 0.7567749368816743, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_public_relations": { - "acc": 0.4, - "acc_stderr": 0.0469237132203465, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.3090909090909091, + "acc_stderr": 0.044262946482000985, + "brier_score": 0.7349220211468389, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_us_foreign_policy": { "acc": 0.4, "acc_stderr": 0.04923659639173309, - "timestamp": "2024-11-19T19-17-59.185280" + "brier_score": 0.7344245727291141, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_sociology": { - "acc": 0.24875621890547264, - "acc_stderr": 0.030567675938916707, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.26865671641791045, + "acc_stderr": 0.031343283582089536, + "brier_score": 0.7514521170912215, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_high_school_macroeconomics": { - "acc": 0.3564102564102564, - "acc_stderr": 0.0242831405294673, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.30256410256410254, + "acc_stderr": 0.02329088805377273, + "brier_score": 0.7377887604031166, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_security_studies": { - "acc": 0.24489795918367346, - "acc_stderr": 0.027529637440174913, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.2938775510204082, + "acc_stderr": 0.029162738410249786, + "brier_score": 0.7441710476153711, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_professional_psychology": { - "acc": 0.2696078431372549, - "acc_stderr": 0.017952449196987866, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.32679738562091504, + "acc_stderr": 0.01897542792050722, + "brier_score": 0.7451067887273528, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_human_sexuality": { "acc": 0.2595419847328244, "acc_stderr": 0.03844876139785271, - "timestamp": "2024-11-19T19-17-59.185280" + "brier_score": 0.7491208222192818, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_econometrics": { - "acc": 0.2807017543859649, - "acc_stderr": 0.042270544512321984, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.32456140350877194, + "acc_stderr": 0.044045561573747685, + "brier_score": 0.7515400174292252, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_miscellaneous": { - "acc": 0.367816091954023, - "acc_stderr": 0.01724382889184626, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.36015325670498083, + "acc_stderr": 0.017166362471369295, + "brier_score": 0.7287497853643151, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_marketing": { - "acc": 0.358974358974359, - "acc_stderr": 0.031426169937919246, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.31196581196581197, + "acc_stderr": 0.03035152732334495, + "brier_score": 0.7373180884734252, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_management": { - "acc": 0.2524271844660194, - "acc_stderr": 0.04301250399690878, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.21359223300970873, + "acc_stderr": 0.040580420156460344, + "brier_score": 0.7665639456583346, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_nutrition": { - "acc": 0.3202614379084967, - "acc_stderr": 0.026716118380156847, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.2875816993464052, + "acc_stderr": 0.02591780611714716, + "brier_score": 0.7541034532974272, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_medical_genetics": { "acc": 0.3, "acc_stderr": 0.046056618647183814, - "timestamp": "2024-11-19T19-17-59.185280" + "brier_score": 0.7361997183161302, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_human_aging": { - "acc": 0.273542600896861, - "acc_stderr": 0.029918586707798827, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.3632286995515695, + "acc_stderr": 0.03227790442850499, + "brier_score": 0.7280599282684578, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_professional_medicine": { - "acc": 0.24632352941176472, - "acc_stderr": 0.02617343857052, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.16911764705882354, + "acc_stderr": 0.022770868010113018, + "brier_score": 0.76751002970653, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_college_medicine": { - "acc": 0.3236994219653179, - "acc_stderr": 0.0356760379963917, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.26011560693641617, + "acc_stderr": 0.03345036916788991, + "brier_score": 0.7487159013561581, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_business_ethics": { - "acc": 0.34, - "acc_stderr": 0.04760952285695236, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.32, + "acc_stderr": 0.046882617226215034, + "brier_score": 0.7461751890143767, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_clinical_knowledge": { - "acc": 0.39622641509433965, - "acc_stderr": 0.030102793781791197, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.32452830188679244, + "acc_stderr": 0.028815615713432104, + "brier_score": 0.741708963885023, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_global_facts": { - "acc": 0.33, - "acc_stderr": 0.04725815626252605, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.35, + "acc_stderr": 0.0479372485441102, + "brier_score": 0.728265985349061, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_virology": { - "acc": 0.35542168674698793, - "acc_stderr": 0.03726214354322415, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.3072289156626506, + "acc_stderr": 0.035915667978246635, + "brier_score": 0.7483456929513479, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_professional_accounting": { - "acc": 0.26595744680851063, - "acc_stderr": 0.026358065698880585, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.2765957446808511, + "acc_stderr": 0.026684564340460987, + "brier_score": 0.753348568316132, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_college_physics": { - "acc": 0.18627450980392157, - "acc_stderr": 0.03873958714149354, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.11764705882352941, + "acc_stderr": 0.03205907733144526, + "brier_score": 0.7722211942164457, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_high_school_physics": { - "acc": 0.2582781456953642, - "acc_stderr": 0.035737053147634576, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.31788079470198677, + "acc_stderr": 0.03802039760107903, + "brier_score": 0.7402329444666135, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_high_school_biology": { "acc": 0.3096774193548387, - "acc_stderr": 0.026302774983517414, - "timestamp": "2024-11-19T19-17-59.185280" + "acc_stderr": 0.026302774983517418, + "brier_score": 0.7455413082159084, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_college_biology": { - "acc": 0.3194444444444444, - "acc_stderr": 0.03899073687357335, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.2708333333333333, + "acc_stderr": 0.03716177437566018, + "brier_score": 0.7436999721198967, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_anatomy": { - "acc": 0.32592592592592595, - "acc_stderr": 0.040491220417025055, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.28888888888888886, + "acc_stderr": 0.0391545063041425, + "brier_score": 0.7370266037272393, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_college_chemistry": { "acc": 0.24, - "acc_stderr": 0.04292346959909283, - "timestamp": "2024-11-19T19-17-59.185280" + "acc_stderr": 0.04292346959909282, + "brier_score": 0.7552559560723006, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_computer_security": { - "acc": 0.34, - "acc_stderr": 0.04760952285695235, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.37, + "acc_stderr": 0.048523658709391, + "brier_score": 0.7338082445157076, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_college_computer_science": { - "acc": 0.31, - "acc_stderr": 0.04648231987117316, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.27, + "acc_stderr": 0.0446196043338474, + "brier_score": 0.7521525063788139, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_astronomy": { - "acc": 0.24342105263157895, - "acc_stderr": 0.034923496688842384, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.3157894736842105, + "acc_stderr": 0.037827289808654685, + "brier_score": 0.750488644502129, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_college_mathematics": { - "acc": 0.29, - "acc_stderr": 0.04560480215720684, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "brier_score": 0.7377399289135675, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_conceptual_physics": { - "acc": 0.3191489361702128, - "acc_stderr": 0.03047297336338006, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.30638297872340425, + "acc_stderr": 0.030135906478517563, + "brier_score": 0.7397898067329776, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_abstract_algebra": { - "acc": 0.28, - "acc_stderr": 0.04512608598542127, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "brier_score": 0.7618169165467457, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_high_school_computer_science": { - "acc": 0.3, - "acc_stderr": 0.046056618647183814, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.33, + "acc_stderr": 0.047258156262526045, + "brier_score": 0.7296516482112015, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_machine_learning": { - "acc": 0.24107142857142858, - "acc_stderr": 0.04059867246952685, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.1875, + "acc_stderr": 0.0370468111477387, + "brier_score": 0.7644975489531513, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_high_school_chemistry": { - "acc": 0.2561576354679803, - "acc_stderr": 0.0307127300709826, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.2512315270935961, + "acc_stderr": 0.030516530732694436, + "brier_score": 0.7448182748498786, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_high_school_statistics": { - "acc": 0.3888888888888889, - "acc_stderr": 0.033247089118091176, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.2777777777777778, + "acc_stderr": 0.03054674526495318, + "brier_score": 0.748414786538635, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_elementary_mathematics": { - "acc": 0.2724867724867725, - "acc_stderr": 0.022930973071633335, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.25925925925925924, + "acc_stderr": 0.022569897074918424, + "brier_score": 0.75592354186923, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_electrical_engineering": { - "acc": 0.3103448275862069, - "acc_stderr": 0.03855289616378948, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.27586206896551724, + "acc_stderr": 0.037245636197746325, + "brier_score": 0.744755125515667, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "mmlu_high_school_mathematics": { - "acc": 0.24814814814814815, - "acc_stderr": 0.026335739404055803, - "timestamp": "2024-11-19T19-17-59.185280" + "acc": 0.29259259259259257, + "acc_stderr": 0.027738969632176088, + "brier_score": 0.7486140129083726, + "brier_score_stderr": "N/A", + "timestamp": "2024-12-15T19-56-45.007907" }, "arc_challenge": { "acc": 0.4334470989761092,