Commit 1e7f7408 authored by fabian.krueger's avatar fabian.krueger
Browse files

Merge branch 'main' of https://git.scc.kit.edu/ng3223/ptsfc_results into main

parents b5fa7195 aa75c87f
"forecast_date","target","horizon","q0.025","q0.25","q0.5","q0.75","q0.975"
2021-11-17,"temperature","36 hour",7.19445777758113,8.58424426119437,9.31346676376485,10.0426892663353,11.4324757499486
2021-11-17,"temperature","48 hour",7.68016406677736,9.02638425728935,9.73274747776319,10.439110698237,11.785330888749
2021-11-17,"temperature","60 hour",9.71872436061509,11.382253556832,12.2551091629756,13.1279647691192,14.7914939653361
2021-11-17,"temperature","72 hour",8.6803283049206,10.2090207903908,11.0111274255172,11.8132340606437,13.3419265461139
2021-11-17,"temperature","84 hour",9.13145269302764,10.9060480296953,11.8371801623739,12.7683122950525,14.5429076317201
"forecast_date","target","horizon","q0.025","q0.25","q0.5","q0.75","q0.975"
2021-11-17,"temperature","36 hour",7.61925,8.4325,8.655,9.0525,9.70275
2021-11-17,"temperature","48 hour",8.25075,8.8775,9.12,9.57,10.321
2021-11-17,"temperature","60 hour",10.13575,11.3275,11.87,12.1625,12.7765
2021-11-17,"temperature","72 hour",9.157,9.9975,10.58,11.0825,11.628
2021-11-17,"temperature","84 hour",10.04475,10.875,11.275,11.6775,12.35575
"forecast_date","target","horizon","q0.025","q0.25","q0.5","q0.75","q0.975"
2021-11-17,"wind","36 hour",13.7882216526812,18.1525188718824,20.4424704607395,22.7324220514782,27.0967193117186
2021-11-17,"wind","48 hour",16.7818671598503,21.0548694270682,23.2969184350026,25.5389674429395,29.8119697102133
2021-11-17,"wind","60 hour",18.1077129922569,22.1261370645464,24.234608612468,26.3430801603896,30.3615042326794
2021-11-17,"wind","72 hour",14.6185314577549,18.873744891138,21.1064600802171,23.3391752694887,27.594388707069
2021-11-17,"wind","84 hour",12.8283089380059,17.9747330399859,20.675070546217,23.3754083618932,28.5218392126887
"forecast_date","target","horizon","q0.025","q0.25","q0.5","q0.75","q0.975"
2021-11-17,"wind","36 hour",14.274,15.7425,17.48,18.75,20.25925
2021-11-17,"wind","48 hour",16.1175,18.3875,19.15,20.2575,21.05025
2021-11-17,"wind","60 hour",18.8375,19.585,21.085,21.8625,22.80025
2021-11-17,"wind","72 hour",15.47475,16.5475,17.605,18.755,19.957
2021-11-17,"wind","84 hour",12.82375,16.455,17.525,19.1175,21.804
rm(list = ls())
library(dplyr)
library(knitr)
setwd("/home/johannes/Documents/Teaching/Ensemble_Seminar/ptsfc_results/evaluation")
# load all evaluation files, append to each other
lf <- list.files() %>% sort
# weather variables not covered on Oct 27
dat <- read.csv(lf[1]) %>% filter(target == "DAX")
for (jj in lf[-1]){
dat <- rbind(dat, read.csv(jj))
}
# omit ensemble/benchmark models from ranking
dat <- dat %>%
filter(!grepl("ensemble", model),
!grepl("EMOS", model), !grepl("benchmark", model))
# select relevant cols, omit NAs (= cases where truth data is not yet available)
ranking <- dat %>% select(model, target, horizon, mean_qscore) %>% na.omit %>%
# compute mean scores sepearately for each target and horizon
group_by(model, target, horizon) %>%
summarise(n = n(), mean_qscore = mean(mean_qscore)) %>%
# compute ranking for each target and horizon
ungroup %>% group_by(target, horizon) %>%
mutate(rk = rank(mean_qscore)) %>%
# compute average rank for each participant
ungroup %>% group_by(model) %>%
# summary information
# nr of evaluation cases as double check
# mean ranks separately for each target and overall
summarise(n_eval_cases = sum(n),
mean_rk_wind = mean(rk[target == "wind"]),
mean_rk_temp = mean(rk[target == "temperature"]),
mean_rk_dax = mean(rk[target == "DAX"]),
mean_rk_overall = mean(rk)) %>%
ungroup
if (length(unique(ranking$n_eval_cases)) != 1){
stop("Nr of evaluation cases differs across participants\n
Perhaps score imputation didn't work properly?")
}
kable(ranking[,-2], digits = 2)
write.csv(ranking, file = "../ptsfc_viz/plot_data/rankings.csv", row.names = FALSE)
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
......@@ -5,6 +5,7 @@ AryaStark
ChandlerBing
Bronn
CaptainRaymondHolt
DAX_benchmark
DexterJettster
DougJudy
GeneralGrievous
......
"model","n_eval_cases","mean_rk_wind","mean_rk_temp","mean_rk_dax","mean_rk_overall"
"AryaStark",34,7.5,9,8,8.16666666666667
"Bronn",34,16.8,15.8,20.2,17.6
"CaptainRaymondHolt",34,7,8.4,11,8.8
"ChandlerBing",34,9.8,5.8,7,7.53333333333333
"DexterJettster",34,16.4,10,3.2,9.86666666666667
"DougJudy",34,9.2,9.4,6.8,8.46666666666667
"GeneralGrievous",34,19.4,20.6,11.6,17.2
"GinaLinetti",34,21,1.8,15.4,12.7333333333333
"HanSolo",34,2.4,19.2,13.4,11.6666666666667
"HotPie",34,11.6,5.8,6,7.8
"JabbaTheHutt",34,11.6,11,6.6,9.73333333333333
"Joey",34,8.2,11,11.8,10.3333333333333
"Joffrey_Baratheon",34,19,8.6,16.4,14.6666666666667
"KyloRen",34,8,6.8,12.4,9.06666666666667
"ObiWanKenobi",34,8.2,9,10,9.06666666666667
"PhoebeBuffay",34,3.4,13.2,6.6,7.73333333333333
"RossGeller",34,15.6,18.6,15.2,16.4666666666667
"SamwellTarly",34,16.6,15.4,20.6,17.5333333333333
"Shaggydog",34,6,13.2,17.2,12.1333333333333
"UglyNakedGuy",34,5.8,9.4,6.2,7.13333333333333
"Yoda",34,7.5,9,5.4,7.3
This diff is collapsed.
"time","value"
2021-06-18 00:00:00,23.7
2021-06-18 12:00:00,33.4
2021-06-19 00:00:00,23.7
2021-06-19 12:00:00,35.3
2021-06-20 00:00:00,22.7
2021-06-20 12:00:00,33.2
2021-06-21 00:00:00,24.1
......@@ -299,3 +295,7 @@
2021-11-13 12:00:00,6.7
2021-11-14 00:00:00,5.5
2021-11-14 12:00:00,7.2
2021-11-15 00:00:00,5.9
2021-11-15 12:00:00,5.9
2021-11-16 00:00:00,5.3
2021-11-16 12:00:00,7.7
"time","value"
2021-06-16 12:00:00,10.08
2021-06-17 00:00:00,8.28
2021-06-17 12:00:00,14.4
2021-06-18 00:00:00,9
2021-06-18 12:00:00,19.08
2021-06-19 00:00:00,7.56
2021-06-19 12:00:00,9
......@@ -299,3 +295,7 @@
2021-11-13 12:00:00,8.64
2021-11-14 00:00:00,2.52
2021-11-14 12:00:00,6.48
2021-11-15 00:00:00,12.96
2021-11-15 12:00:00,13.68
2021-11-16 00:00:00,10.08
2021-11-16 12:00:00,8.64
......@@ -24,6 +24,9 @@ source("functions.R")
# summary of evaluations:
eval <- read.csv("plot_data/summary_eval.csv")
# rankings:
rankings <- read.csv("plot_data/rankings.csv")
rankings$mean_rk_overall <- round(rankings$mean_rk_overall, 2)
# get vector of model names:
dat_models <- read.csv("plot_data/list_teams.csv")
......@@ -386,4 +389,11 @@ shinyServer(function(input, output, session) {
colnames = c("", "model", "target", "horizon", "MAE", "MQS", "C0.5", "C0.95", "n", "imputed"))
})
# table with ranks:
# output table:
output$tab_rankings <- DT::renderDataTable({
datatable(rankings[, -2], filter = "top",
colnames = c("", "model", "av. rank wind", "av. rank temperature", "av. rank DAX", "av. rank overall"))
})
})
......@@ -62,7 +62,9 @@ shinyUI(fluidPage(
h4("Forecast visualization"),
plotlyOutput("tsplot", height = "600px"),
h4("Average evaluation scores by target and horizon"),
dataTableOutput("tab")
dataTableOutput("tab"),
h4("Rankings"),
dataTableOutput("tab_rankings")
)
)
))
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment