Merge branch 'main' of https://git.scc.kit.edu/ng3223/ptsfc_results

.gitignore

+.Rhistory

20211020/20211020_DAX_benchmark.csv

+"forecast_date","target","horizon","q0.025","q0.25","q0.5","q0.75","q0.975"
+"2021-10-20","DAX","1 day",-2.65844263691718,-0.485396555504147,0.0657882169970314,0.652898469731245,2.22946787319034
+"2021-10-20","DAX","2 day",-3.76658704113496,-0.754702332963619,0.13670459761812,0.9054642766952,3.10108794007109
+"2021-10-20","DAX","5 day",-4.73040567059547,-0.849379110212389,0.143043446923219,1.18607119760754,3.89032640718187
+"2021-10-20","DAX","6 day",-5.23203014061953,-0.968345906220103,0.193504753259699,1.39125836779996,4.57407374600166
+"2021-10-20","DAX","7 day",-5.68685922947709,-1.09164649900868,0.346636456527527,1.48817956141838,5.19223708311003

20211027/20211027_DAX_benchmark.csv

+"forecast_date","target","horizon","q0.025","q0.25","q0.5","q0.75","q0.975"
+"2021-10-27","DAX","1 day",-2.65844263691718,-0.483787003227087,0.0693890771191086,0.656902293272221,2.22946787319034
+"2021-10-27","DAX","2 day",-3.76658704113496,-0.754702332963619,0.137166393470345,0.9054642766952,3.10108794007109
+"2021-10-27","DAX","5 day",-4.73040567059547,-0.849379110212389,0.14578755206589,1.18415337595774,3.89032640718187
+"2021-10-27","DAX","6 day",-5.23203014061953,-0.968345906220103,0.197265375858358,1.37748695784952,4.57407374600166
+"2021-10-27","DAX","7 day",-5.68685922947709,-1.09164649900868,0.346067477430712,1.4646373891495,5.19223708311003

20211103/20211103_DAX_benchmark.csv

+"forecast_date","target","horizon","q0.025","q0.25","q0.5","q0.75","q0.975"
+"2021-11-03","DAX","1 day",-2.65844263691718,-0.481709571803846,0.0733950537846084,0.662320554605333,2.22946787319034
+"2021-11-03","DAX","2 day",-3.76658704113496,-0.732473649417953,0.142980430012329,0.907244580165889,3.10108794007109
+"2021-11-03","DAX","5 day",-4.73040567059547,-0.81103402646141,0.16179836626895,1.18916624608452,3.89032640718187
+"2021-11-03","DAX","6 day",-5.23203014061953,-0.933589441129046,0.219661627911449,1.39266918407004,4.57407374600166
+"2021-11-03","DAX","7 day",-5.68685922947709,-1.06110544198859,0.357564327371396,1.47103419000207,5.19223708311003

20211103/20211103_mean_ensemble.csv

 "","forecast_date","target","horizon","q0.025","q0.25","q0.5","q0.75","q0.975"
-"1",2021-11-03,"DAX","1 day",-2.02825713270737,-0.382000564657368,0.106464667575488,0.633568047304173,1.86431768894998
-"2",2021-11-03,"DAX","2 day",-2.74697146311731,-0.609582612279664,0.160395960037263,0.904187315362982,2.60681420123488
-"3",2021-11-03,"temperature","36 hour",6.59747367469965,7.75563311186704,8.69887807534744,9.52763875168277,11.3926145477422
-"4",2021-11-03,"wind","36 hour",9.07185715917746,13.9724340527979,16.5228790836859,19.181945985747,24.0967341855328
-"5",2021-11-03,"temperature","48 hour",5.98809323105581,7.34347901814137,8.24353647840509,9.15011171909121,10.9056253262549
-"6",2021-11-03,"wind","48 hour",7.43413200568977,13.3414804779109,16.6912177447552,20.1301909785707,26.9824161773462
-"7",2021-11-03,"DAX","5 day",-3.31095368892838,-0.681454281132449,0.201764770470663,1.09414898031902,3.19593331830434
-"8",2021-11-03,"DAX","6 day",-3.5566096458164,-0.732581817607553,0.2518118710017,1.19761058309121,3.44599521183298
-"9",2021-11-03,"temperature","60 hour",7.95196554993163,9.33213318402821,10.2363931643125,11.1404391024969,13.0767705729913
-"10",2021-11-03,"wind","60 hour",9.86108377920895,13.9033742219242,16.106508200007,18.3011379485107,22.7910787365034
-"11",2021-11-03,"DAX","7 day",-3.9820989620208,-0.688418980304888,0.377530351737436,1.36027341796101,3.84868100382188
-"12",2021-11-03,"temperature","72 hour",4.80510320481326,6.39939741130075,7.28302287423084,8.30430009028192,10.3134608538387
-"13",2021-11-03,"wind","72 hour",7.06710961829704,11.8294061744475,14.0728386063238,16.7546550740087,21.965993475498
-"14",2021-11-03,"temperature","84 hour",8.10358735206451,9.85978069774847,10.8139403988423,11.901646679211,14.1055382821453
-"15",2021-11-03,"wind","84 hour",10.0861537991543,14.9879076376588,17.597066087642,20.1115597488391,24.9987728277514
+"1",2021-11-03,"DAX","1 day",-2.02825898441139,-0.382001114867622,0.10646219401679,0.633568222797302,1.86430620015501
+"2",2021-11-03,"DAX","2 day",-2.74695666220728,-0.609587052303678,0.160394206437615,0.904187452769997,2.60680672504881
+"3",2021-11-03,"temperature","36 hour",6.58620605884461,7.7506624957451,8.68378920482086,9.50701099823736,11.3500111396742
+"4",2021-11-03,"wind","36 hour",9.04714733905533,13.9751781342818,16.529877924591,19.2028606263171,24.1361491198693
+"5",2021-11-03,"temperature","48 hour",5.9984966070696,7.35573287499317,8.24920588295004,9.14893596011242,10.890294935311
+"6",2021-11-03,"wind","48 hour",7.35974664186218,13.3216369884164,16.7115266126322,20.1742319234263,27.0266024502523
+"7",2021-11-03,"DAX","5 day",-3.31093102313141,-0.681457329407887,0.201764033474751,1.09415412610773,3.19590694803525
+"8",2021-11-03,"DAX","6 day",-3.55656715945281,-0.732380411676226,0.251937471326226,1.19760493702535,3.44623593877625
+"9",2021-11-03,"temperature","60 hour",7.95356347772432,9.33802823170524,10.2373395363912,11.1364453601363,13.0548273975841
+"10",2021-11-03,"wind","60 hour",9.8362678914806,13.8965251671767,16.1051717120068,18.3069123196153,22.7969139267778
+"11",2021-11-03,"DAX","7 day",-3.98233713943126,-0.688140361611747,0.377582682970036,1.36055346804885,3.85014046324557
+"12",2021-11-03,"temperature","72 hour",4.81460433233148,6.40831686470388,7.28975322128959,8.30333526087144,10.2953196880303
+"13",2021-11-03,"wind","72 hour",7.01753841107474,11.8130856280002,14.0781243592871,16.7708662808547,22.0051857623602
+"14",2021-11-03,"temperature","84 hour",8.10517745576832,9.86660792726265,10.8222082295159,11.906013247873,14.0972340463277
+"15",2021-11-03,"wind","84 hour",10.0523076471881,14.9813668508167,17.5948664518429,20.1178464249406,25.0170925090548

20211103/20211103_median_ensemble.csv

@@ -6,10 +6,10 @@
 "5",2021-11-03,"temperature","48 hour",6.24817763140052,7.64982543943644,8.38527159202891,9.12071774462139,10.5223655526573
 "6",2021-11-03,"wind","48 hour",5.57449791,12.8453932405484,17.1989394416795,21.2312145999616,28.087073
 "7",2021-11-03,"DAX","5 day",-3.04769045711534,-0.716860000349991,0.193204562953042,1.1539173162404,2.88964264274188
-"8",2021-11-03,"DAX","6 day",-3.63853664521561,-0.782593804660249,0.237005862015918,1.1977347965402,3.00809094327631
+"8",2021-11-03,"DAX","6 day",-3.63802382151669,-0.780501133717677,0.23829450651625,1.19767268981571,3.01097030020352
 "9",2021-11-03,"temperature","60 hour",7.99191374474882,9.47950937595391,10.26005246628,11.0405955434807,12.5281911878111
 "10",2021-11-03,"wind","60 hour",9.24068658600017,13.7321478532379,16.073096,18.4454972261262,22.9369584933639
-"11",2021-11-03,"DAX","7 day",-3.94445156512591,-0.755070977268125,0.365553059150531,1.3541123160286,3.60180096497218
+"11",2021-11-03,"DAX","7 day",-3.94690480781968,-0.752178574485322,0.36609577980414,1.35719286699481,3.61684968248609
 "12",2021-11-03,"temperature","72 hour",5.04263139276858,6.62238374637901,7.45128155069957,8.28017935502012,9.85993170863055
 "13",2021-11-03,"wind","72 hour",5.82782943773967,11.4213925132652,14.2049824304059,17.1599352451589,22.9458006470541
 "14",2021-11-03,"temperature","84 hour",8.14333994465985,10.0304614356028,11.0206361656824,12.0108108957619,13.8979323867049

20211110/20211110_DAX_benchmark.csv

+"forecast_date","target","horizon","q0.025","q0.25","q0.5","q0.75","q0.975"
+"2021-11-10","DAX","1 day",-2.65844263691718,-0.478408853013734,0.0733950537846084,0.65948147757644,2.22946787319034
+"2021-11-10","DAX","2 day",-3.76658704113496,-0.732473649417953,0.152264060287965,0.905783297834439,3.10108794007109
+"2021-11-10","DAX","5 day",-4.73040567059547,-0.81103402646141,0.16179836626895,1.1996376118808,3.89032640718187
+"2021-11-10","DAX","6 day",-5.23203014061953,-0.933589441129046,0.23998729150474,1.39631925115538,4.57407374600166
+"2021-11-10","DAX","7 day",-5.68685922947709,-1.06110544198859,0.374569351209342,1.49206818774594,5.19223708311003

20211110/20211110_mean_ensemble.csv

+"","forecast_date","target","horizon","q0.025","q0.25","q0.5","q0.75","q0.975"
+"1",2021-11-10,"DAX","1 day",-1.97839850611856,-0.36983952534121,0.0701211299469397,0.53581584754644,1.74398192448053
+"2",2021-11-10,"DAX","2 day",-2.7193145003102,-0.538946794723462,0.148834694736427,0.79682840120006,2.47024177839926
+"3",2021-11-10,"temperature","36 hour",6.50655987306001,8.12114810930847,9.10573297651932,9.91391119018101,11.7640742693176
+"4",2021-11-10,"wind","36 hour",4.79698437263621,7.96013393450242,9.7228883949328,11.5474214911919,15.3204010866751
+"5",2021-11-10,"temperature","48 hour",3.45439583480986,5.91943919825621,7.14274948953963,8.12442440865849,10.2770226998875
+"6",2021-11-10,"wind","48 hour",1.61258692485324,3.90272093411068,5.71406211904285,7.6883331958237,11.5970055588307
+"7",2021-11-10,"DAX","5 day",-3.34714703027757,-0.615928645876698,0.228684026447029,1.04990747716939,3.31994803626036
+"8",2021-11-10,"DAX","6 day",-3.47483687949756,-0.688982773350172,0.255115288770901,1.19162368538956,3.40004221393911
+"9",2021-11-10,"temperature","60 hour",6.17913893578767,8.44646391461461,9.574783802297,10.7027809753665,12.9284491890783
+"10",2021-11-10,"wind","60 hour",4.76804546409593,8.47415187645485,10.7369752903673,12.9088477860076,17.1082696257816
+"11",2021-11-10,"DAX","7 day",-3.89306329434958,-0.677819442623292,0.385113073794824,1.34727327878675,3.68952973497652
+"12",2021-11-10,"temperature","72 hour",2.11195476357749,3.99000792398846,4.96506438801635,5.81549041963432,7.84378264809856
+"13",2021-11-10,"wind","72 hour",4.10462378331307,7.72433280586531,9.69699714948641,11.9131074853994,16.1687738379047
+"14",2021-11-10,"temperature","84 hour",5.59685910702724,7.72495955024123,8.88213776134557,10.0695933409688,12.4636343223966
+"15",2021-11-10,"wind","84 hour",5.06493429498838,9.56384790006749,11.9820444550826,14.8532966400121,20.0924903522431

20211110/20211110_median_ensemble.csv

+"","forecast_date","target","horizon","q0.025","q0.25","q0.5","q0.75","q0.975"
+"1",2021-11-10,"DAX","1 day",-1.96436473112436,-0.408849110035622,0.0765773585696374,0.542798368407873,1.44063566550546
+"2",2021-11-10,"DAX","2 day",-2.75868968121492,-0.554447948109177,0.149823355537018,0.730664711118278,2.10011927090861
+"3",2021-11-10,"temperature","36 hour",6.67607181137691,8.30414527997504,9.16072594447361,10.0173066089722,11.6498182572664
+"4",2021-11-10,"wind","36 hour",4.34504858003043,7.87298696214459,9.49455891011215,11.5037394501141,15.4876428620055
+"5",2021-11-10,"temperature","48 hour",3.65593427587368,5.90227873620489,7.02087254719095,8.17243767910115,10.2761841419757
+"6",2021-11-10,"wind","48 hour",0.837796615165771,3.10287186479191,5.04733954657358,7.14046761457886,11.2484139593305
+"7",2021-11-10,"DAX","5 day",-2.92843159399775,-0.675840117374157,0.2367524864287,1.07914521209142,2.82198057762175
+"8",2021-11-10,"DAX","6 day",-3.31499158062655,-0.739010905415254,0.269960733111179,1.21460533105313,3.09373509264794
+"9",2021-11-10,"temperature","60 hour",6.27949434869524,8.49468504115018,9.66110835786509,10.82753167458,12.9667893982544
+"10",2021-11-10,"wind","60 hour",4.08381571299951,8.23995523701437,10.6186779260786,13.028093015133,17.3533857458042
+"11",2021-11-10,"DAX","7 day",-3.51113,-0.693450881454349,0.392,1.32872962502132,3.36447402175136
+"12",2021-11-10,"temperature","72 hour",2.14300245574447,3.9234261541212,4.8576164346243,5.79180671512741,7.57223041350414
+"13",2021-11-10,"wind","72 hour",3.5243619310468,7.54767564248681,9.86169160916679,12.3478586077613,17.0906063168625
+"14",2021-11-10,"temperature","84 hour",5.75534789222807,7.92417243300848,9.10065943944641,10.2132106023971,12.3911711803964
+"15",2021-11-10,"wind","84 hour",4.10625,9.4366214312162,12.4299171367345,15.2817855847627,21.0040481545951

20211117/20211117_DAX_benchmark.csv

+"forecast_date","target","horizon","q0.025","q0.25","q0.5","q0.75","q0.975"
+"2021-11-17","DAX","1 day",-2.65844263691718,-0.478408853013734,0.0754703798988565,0.65948147757644,2.22946787319034
+"2021-11-17","DAX","2 day",-3.76658704113496,-0.730638752085122,0.165072327412297,0.907244580165889,3.10108794007109
+"2021-11-17","DAX","5 day",-4.73040567059547,-0.805625742830962,0.168228145854687,1.1996376118808,3.89032640718187
+"2021-11-17","DAX","6 day",-5.23203014061953,-0.928194576204255,0.246671919221697,1.39631925115538,4.57407374600166
+"2021-11-17","DAX","7 day",-5.68685922947709,-1.06110544198859,0.380815682407221,1.49206818774594,5.19223708311003

20211117/20211117_t2m_EMOS.csv

+"forecast_date","target","horizon","q0.025","q0.25","q0.5","q0.75","q0.975"
+2021-11-17,"temperature","36 hour",7.19445777758113,8.58424426119437,9.31346676376485,10.0426892663353,11.4324757499486
+2021-11-17,"temperature","48 hour",7.68016406677736,9.02638425728935,9.73274747776319,10.439110698237,11.785330888749
+2021-11-17,"temperature","60 hour",9.71872436061509,11.382253556832,12.2551091629756,13.1279647691192,14.7914939653361
+2021-11-17,"temperature","72 hour",8.6803283049206,10.2090207903908,11.0111274255172,11.8132340606437,13.3419265461139
+2021-11-17,"temperature","84 hour",9.13145269302764,10.9060480296953,11.8371801623739,12.7683122950525,14.5429076317201

20211117/20211117_t2m_benchmark.csv

+"forecast_date","target","horizon","q0.025","q0.25","q0.5","q0.75","q0.975"
+2021-11-17,"temperature","36 hour",7.61925,8.4325,8.655,9.0525,9.70275
+2021-11-17,"temperature","48 hour",8.25075,8.8775,9.12,9.57,10.321
+2021-11-17,"temperature","60 hour",10.13575,11.3275,11.87,12.1625,12.7765
+2021-11-17,"temperature","72 hour",9.157,9.9975,10.58,11.0825,11.628
+2021-11-17,"temperature","84 hour",10.04475,10.875,11.275,11.6775,12.35575

20211117/20211117_wind_EMOS.csv

+"forecast_date","target","horizon","q0.025","q0.25","q0.5","q0.75","q0.975"
+2021-11-17,"wind","36 hour",13.7882216526812,18.1525188718824,20.4424704607395,22.7324220514782,27.0967193117186
+2021-11-17,"wind","48 hour",16.7818671598503,21.0548694270682,23.2969184350026,25.5389674429395,29.8119697102133
+2021-11-17,"wind","60 hour",18.1077129922569,22.1261370645464,24.234608612468,26.3430801603896,30.3615042326794
+2021-11-17,"wind","72 hour",14.6185314577549,18.873744891138,21.1064600802171,23.3391752694887,27.594388707069
+2021-11-17,"wind","84 hour",12.8283089380059,17.9747330399859,20.675070546217,23.3754083618932,28.5218392126887

20211117/20211117_wind_benchmark.csv

+"forecast_date","target","horizon","q0.025","q0.25","q0.5","q0.75","q0.975"
+2021-11-17,"wind","36 hour",14.274,15.7425,17.48,18.75,20.25925
+2021-11-17,"wind","48 hour",16.1175,18.3875,19.15,20.2575,21.05025
+2021-11-17,"wind","60 hour",18.8375,19.585,21.085,21.8625,22.80025
+2021-11-17,"wind","72 hour",15.47475,16.5475,17.605,18.755,19.957
+2021-11-17,"wind","84 hour",12.82375,16.455,17.525,19.1175,21.804

code/compute_ranking.R

+rm(list = ls())
+
+library(dplyr)
+library(knitr)
+
+setwd("/home/johannes/Documents/Teaching/Ensemble_Seminar/ptsfc_results/evaluation")
+
+# load all evaluation files, append to each other
+lf <- list.files() %>% sort
+# weather variables not covered on Oct 27
+dat <- read.csv(lf[1]) %>% filter(target == "DAX")
+for (jj in lf[-1]){
+  dat <- rbind(dat, read.csv(jj))
+}
+# omit ensemble/benchmark models from ranking 
+dat <- dat %>% 
+  filter(!grepl("ensemble", model), 
+         !grepl("EMOS", model), !grepl("benchmark", model))
+
+# select relevant cols, omit NAs (= cases where truth data is not yet available)
+ranking <- dat %>% select(model, target, horizon, mean_qscore) %>% na.omit %>%
+  # compute mean scores sepearately for each target and horizon
+  group_by(model, target, horizon) %>% 
+  summarise(n = n(), mean_qscore = mean(mean_qscore)) %>%
+  # compute ranking for each target and horizon
+  ungroup %>% group_by(target, horizon) %>%
+  mutate(rk = rank(mean_qscore)) %>%
+  # compute average rank for each participant
+  ungroup %>% group_by(model) %>% 
+  # summary information
+  # nr of evaluation cases as double check
+  # mean ranks separately for each target and overall
+  summarise(n_eval_cases = sum(n),
+            mean_rk_wind = mean(rk[target == "wind"]),
+            mean_rk_temp = mean(rk[target == "temperature"]),
+            mean_rk_dax = mean(rk[target == "DAX"]),
+            mean_rk_overall = mean(rk)) %>%
+  ungroup
+
+if (length(unique(ranking$n_eval_cases)) != 1){
+  stop("Nr of evaluation cases differs across participants\n
+        Perhaps score imputation didn't work properly?")
+}
+
+kable(ranking[,-2], digits = 2)
+
+write.csv(ranking, file = "../ptsfc_viz/plot_data/rankings.csv", row.names = FALSE)

ptsfc_viz/plot_data/list_teams.csv

@@ -5,6 +5,7 @@ AryaStark
 ChandlerBing
 Bronn
 CaptainRaymondHolt
+DAX_benchmark
 DexterJettster
 DougJudy
 GeneralGrievous