add ccs csv eval

.gitignore

@@ -1,4 +1,4 @@
-*.csv
+# *.csv
 *.npy
 elk/models/*
 elk/trained/*

elk-reporters/microsoft/deberta-v2-xxlarge-mnli/imdb/eloquent-hofstadter/eval.csv

@@ -0,0 +1,49 @@
+layer,pseudo_auroc,train_loss,acc,cal_acc,auroc,ece,lm_auroc,lm_acc,lr_auroc,lr_acc
+0.0,0.5,0.19050443172454834,0.5286153554916382,0.5286153554916382,0.5057141775147929,0.4997003674507141,,,0.7559683786982249,0.6856923076923077
+1.0,0.5,0.19998577237129211,0.501230776309967,0.5089230537414551,0.5095047573964497,5.960464477539063e-08,,,0.5332756923076923,0.5
+2.0,0.5,0.20001238584518433,0.5098461508750916,0.5049231052398682,0.5101856568047337,0.4082795977592468,,,0.5048297278106508,0.5
+3.0,0.5,0.2000013142824173,0.5479999780654907,0.5593846440315247,0.563173301775148,0.4999985098838806,,,0.5178939644970413,0.5
+4.0,0.5,0.1999962478876114,0.5264615416526794,0.5270769596099854,0.5397570650887574,1.430511474609375e-06,,,0.5083158816568047,0.5010769230769231
+5.0,0.5,0.19998642802238464,0.5070769190788269,0.508307695388794,0.511376473372781,0.4999994933605194,,,0.5967475029585799,0.5612307692307692
+6.0,0.5,0.19999566674232483,0.5292307734489441,0.5341538190841675,0.5420620118343196,0.4082765281200409,,,0.5719660591715976,0.5564615384615385
+7.0,0.5,0.20000283420085907,0.7590768933296204,0.7581538558006287,0.8323321183431954,0.4082726538181305,,,0.6113397396449705,0.582
+8.0,0.5,0.1998833566904068,0.8132307529449463,0.812000036239624,0.8936160946745562,4.112720489501953e-06,,,0.6749464142011834,0.62
+9.0,0.5,0.20001041889190674,0.7492307424545288,0.7492307424545288,0.8403203786982248,5.960464477539063e-08,,,0.5605344852071006,0.5441538461538461
+10.0,0.5,0.20000821352005005,0.5393846035003662,0.5418461561203003,0.5704789585798817,0.4996912479400635,,,0.5366632899408283,0.5
+11.0,0.5,0.19903604686260223,0.7258461713790894,0.726769208908081,0.8088378698224852,2.1219253540039062e-05,,,0.7972800473372781,0.7241538461538461
+12.0,0.5,0.19882112741470337,0.665538489818573,0.6670769453048706,0.7076866272189348,0.44655469059944153,,,0.7225216094674557,0.6544615384615384
+13.0,0.5,0.200009286403656,0.5633845925331116,0.5624615550041199,0.5824763076923078,0.4082789123058319,,,0.5284576568047336,0.5
+14.0,0.5,0.20000828802585602,0.6399999856948853,0.6381538510322571,0.6921813964497041,5.960464477539063e-08,,,0.5843580591715977,0.554923076923077
+15.0,0.5,0.2000107765197754,0.6670769453048706,0.6689230799674988,0.7491804023668639,0.4472118318080902,,,0.53783550295858,0.5
+16.0,0.5,0.2000112235546112,0.5427692532539368,0.5467692017555237,0.5586410414201184,0.4999997317790985,,,0.5272034556213019,0.5
+17.0,0.5,0.1999216079711914,0.7670769095420837,0.7698461413383484,0.8352007573964497,0.4082171320915222,,,0.7176580828402367,0.6581538461538462
+18.0,0.5,0.20000678300857544,0.7849230766296387,0.790461540222168,0.8680450650887575,0.40827542543411255,,,0.6429947928994083,0.6003076923076923
+19.0,0.5,0.1999843716621399,0.7975384593009949,0.7969231009483337,0.8941947455621302,0.4082564115524292,,,0.722229301775148,0.6530769230769231
+20.0,0.5,0.19994601607322693,0.8409230709075928,0.835692286491394,0.9180137278106508,1.7285346984863281e-06,,,0.7568755029585799,0.6872307692307692
+21.0,0.5,0.20000657439231873,0.7683076858520508,0.7664615511894226,0.8586809940828403,0.0,,,0.5716189822485207,0.5487692307692308
+22.0,0.5,0.19999736547470093,0.5692307949066162,0.572615385055542,0.5969762840236686,2.980232238769531e-07,,,0.5542461538461538,0.5416923076923077
+23.0,0.5,0.1996442675590515,0.7984615564346313,0.799384593963623,0.8811697988165681,1.6689300537109375e-05,,,0.829723076923077,0.7466153846153846
+24.0,0.5,0.1983022838830948,0.635692298412323,0.635692298412323,0.697231526627219,0.4071957468986511,,,0.7899728284023668,0.7101538461538461
+25.0,0.5,0.20000535249710083,0.7073845863342285,0.7086153626441956,0.778940023668639,0.40827471017837524,,,0.5815162603550296,0.5598461538461539
+26.0,0.5,0.2000066637992859,0.8095384836196899,0.8058461546897888,0.8880768757396449,0.40827444195747375,,,0.6461629822485206,0.6084615384615385
+27.0,0.5,0.2000078707933426,0.7886154055595398,0.7830769419670105,0.8696499881656805,1.1920928955078125e-07,,,0.5927550295857988,0.5638461538461539
+28.0,0.5,0.2000090777873993,0.7338461875915527,0.7335384488105774,0.8191157396449704,5.960464477539063e-08,,,0.6122943905325444,0.5761538461538461
+29.0,0.5,0.199776753783226,0.8326153755187988,0.83323073387146,0.9095954556213017,8.404254913330078e-06,,,0.8113679526627218,0.7266153846153847
+30.0,0.5,0.19996507465839386,0.853538453578949,0.8544615507125854,0.9240433609467456,0.4082392752170563,,,0.7504514556213018,0.6770769230769231
+31.0,0.5,0.1998489499092102,0.8621538281440735,0.8646153807640076,0.9368070059171597,0.40813490748405457,,,0.8421494911242604,0.7516923076923077
+32.0,0.5,0.19875645637512207,0.864307701587677,0.8658461570739746,0.934262721893491,0.4071587920188904,,,0.8565076923076923,0.7669230769230769
+33.0,0.5,0.1986783891916275,0.8544615507125854,0.8596923351287842,0.9320835029585798,0.40708374977111816,,,0.8615651597633136,0.7715384615384615
+34.0,0.5,0.1976706087589264,0.8178461790084839,0.8190768957138062,0.9087589112426036,0.4061126410961151,,,0.8301345798816568,0.7373846153846154
+35.0,0.5,0.16335442662239075,0.8584615588188171,0.8652307987213135,0.937725727810651,0.3688405454158783,,,0.9271376094674556,0.8441538461538461
+36.0,0.5,0.13132089376449585,0.8646153807640076,0.8658461570739746,0.9414301538461538,0.0055495500564575195,,,0.9424764023668638,0.8613846153846154
+37.0,0.5,0.19950412213802338,0.8621538281440735,0.8609230518341064,0.9354168047337278,0.40782442688941956,,,0.8685545088757396,0.7826153846153846
+38.0,0.5,0.19935892522335052,0.8513846397399902,0.8553845882415771,0.9311511479289941,0.4076986312866211,,,0.8672647573964498,0.7783076923076923
+39.0,0.5,0.19936242699623108,0.8553845882415771,0.8584615588188171,0.9303759526627219,0.40769582986831665,,,0.8408650887573965,0.7524615384615385
+40.0,0.5,0.19932380318641663,0.8504615426063538,0.8538461327552795,0.9288202603550296,0.4076540172100067,,,0.8326092307692308,0.7426153846153846
+41.0,0.5,0.19382818043231964,0.8476923108100891,0.853538453578949,0.9284296331360946,0.40250611305236816,,,0.8734713372781066,0.782
+42.0,0.5,0.19857019186019897,0.847076952457428,0.8486154079437256,0.9247568284023668,0.4069719612598419,,,0.8258002840236687,0.7335384615384616
+43.0,0.5,0.19845682382583618,0.8378461599349976,0.8412307500839233,0.9199043786982248,0.4068708121776581,,,0.8409069349112426,0.7509230769230769
+44.0,0.5,0.1855067014694214,0.8335384726524353,0.835692286491394,0.9183272899408284,0.0004011988639831543,,,0.8606814674556212,0.7715384615384615
+45.0,0.5,0.16924627125263214,0.8412307500839233,0.8406153917312622,0.9202624378698224,0.36489197611808777,,,0.8886886153846153,0.7952307692307692
+46.0,0.5,0.17996063828468323,0.8390769362449646,0.8406153917312622,0.9154997396449706,0.37600746750831604,,,0.8477955502958581,0.7575384615384615
+47.0,0.5,0.10626590251922607,0.8578461408615112,0.8541538715362549,0.9314291124260355,0.011760294437408447,,,0.950702201183432,0.8755384615384615

elk/evaluation/evaluate.py

@@ -91,7 +91,8 @@ def evaluate_reporter(
  lr_model = load_baseline(lr_dir, layer)
  lr_model.eval()
  lr_auroc, lr_acc = evaluate_baseline(
- lr_model.cuda(), test_x0.cuda(), test_x1.cuda(), test_labels
+ # lr_model.cuda(), test_x0.cuda(), test_x1.cuda(), test_labels
+ lr_model.to('mps'), test_x0.to('mps'), test_x1.to('mps'), test_labels
  )
 
  stats_row["lr_auroc"] = lr_auroc