Merge branch 'main' of https://git.scc.kit.edu/ng3223/ptsfc_results into main

code/process_viz_data.R

@@ -15,66 +15,72 @@ for(lab in labels_weeks){
   # identify files
   fls <- list.files(paste0("../", lab))
   
-  # run through files and read in:
-  result <- NULL
-  for(i in seq_along(fls)){
-    dat_temp <- read.csv(paste0("../", lab, "/", fls[i]), 
-                         colClasses = c(forecast_date = "Date"))
-    # remove row numbers if necessary:
-    dat_temp$X <- NULL
-    
-    # add model name:
-    dat_temp$model <- extract_name_from_file(fls[i])
-    
-    # add column for target end date / time:
-    dat_temp$target_end_date <- NA
-    
-    # re-format forecast time:
-    dat_temp$forecast_date <- as.POSIXlt(dat_temp$forecast_date, "GMT") + 24*60*60 - 60
-    
-    # handle DAX forecasts:
-    subs_DAX <- subset(dat_temp, target == "DAX")
-    # generate target end date / time:
-    days_numeric <- as.numeric(gsub(" day", "", subs_DAX$horizon))
-    subs_DAX$target_end_date <- subs_DAX$forecast_date + 60 + days_numeric*24*60*60 - 6.5*60*60
+  if(any(grepl("ensemble", fls))){
+    # run through files and read in:
+    result <- NULL
+    for(i in seq_along(fls)){
+      dat_temp <- read.csv(paste0("../", lab, "/", fls[i]), 
+                           colClasses = c(forecast_date = "Date"))
+      # remove row numbers if necessary:
+      dat_temp$X <- NULL
+      
+      # add model name:
+      dat_temp$model <- extract_name_from_file(fls[i])
+      
+      # add column for target end date / time:
+      dat_temp$target_end_date <- NA
+      
+      # re-format forecast time:
+      dat_temp$forecast_date <- as.POSIXlt(dat_temp$forecast_date, "GMT") + 24*60*60 - 60
+      
+      # handle DAX forecasts:
+      subs_DAX <- subset(dat_temp, target == "DAX")
+      # generate target end date / time:
+      days_numeric <- as.numeric(gsub(" day", "", subs_DAX$horizon))
+      subs_DAX$target_end_date <- subs_DAX$forecast_date + 60 + days_numeric*24*60*60 - 6.5*60*60
+      
+      # handle weather forecasts:
+      subs_weather <- subset(dat_temp, target %in% c("wind", "temperature"))
+      # generate target end date / time:
+      hours_numeric <- as.numeric(gsub(" hour", "", subs_weather$horizon))
+      subs_weather$target_end_date <- subs_weather$forecast_date + 60 + hours_numeric*60*60 - 24*60*60
+      
+      # put back together:
+      dat_temp <- rbind(subs_DAX, subs_weather)
+      
+      # add to data.frame:
+      if(!is.null(result)){
+        result <- rbind(result, dat_temp)
+      }else{
+        result <- dat_temp
+      }
+    }
     
-    # handle weather forecasts:
-    subs_weather <- subset(dat_temp, target %in% c("wind", "temperature"))
-    # generate target end date / time:
-    hours_numeric <- as.numeric(gsub(" hour", "", subs_weather$horizon))
-    subs_weather$target_end_date <- subs_weather$forecast_date + 60 + hours_numeric*60*60 - 24*60*60
+    # get evaluation data:
+    dat_eval <- NULL
+    try({
+      dat_eval <- read.csv(paste0("../evaluation/evaluation_", lab, ".csv"))
+    }, silent = TRUE)
     
-    # put back together:
-    dat_temp <- rbind(subs_DAX, subs_weather)
     
-    # add to data.frame:
+    # write out if non-empty
     if(!is.null(result)){
-      result <- rbind(result, dat_temp)
-    }else{
-      result <- dat_temp
+      # add evaluation data:
+      if(!is.null(dat_eval)){
+        dat_eval <- dat_eval[, c("target", "model", "horizon", "mean_qscore", "ae")]
+        result <- merge(result, dat_eval, by = c("model", "target", "horizon"), all.x = TRUE)
+      }else{
+        result$mean_qscore <- result$ae <- NA
+      }
+      
+      # result <- subset(result, target == "DAX" & horizon %in% paste(c(1, 2, 5, 6, 7), "day"))
+      forecast_date <- as.Date(result$forecast_date[1])
+      write.csv(result, file = paste0("../ptsfc_viz/plot_data/plot_data_", forecast_date, ".csv"), row.names = FALSE)
     }
+  }else{
+    warning(paste("Not processing visualization data for ", lab, "as ensemble not created yet."))
   }
   
-  # get evaluation data:
-  dat_eval <- NULL
-  try({
-    dat_eval <- read.csv(paste0("../evaluation/evaluation_", lab, ".csv"))
-  }, silent = TRUE)
-
   
-  # write out if non-empty
-  if(!is.null(result)){
-    # add evaluation data:
-    if(!is.null(dat_eval)){
-      dat_eval <- dat_eval[, c("target", "model", "horizon", "mean_qscore", "ae")]
-      result <- merge(result, dat_eval, by = c("model", "target", "horizon"), all.x = TRUE)
-    }else{
-      result$mean_qscore <- result$ae <- NA
-    }
-    
-    # result <- subset(result, target == "DAX" & horizon %in% paste(c(1, 2, 5, 6, 7), "day"))
-    forecast_date <- as.Date(result$forecast_date[1])
-    write.csv(result, file = paste0("../ptsfc_viz/plot_data/plot_data_", forecast_date, ".csv"), row.names = FALSE)
-  }
 }
 

ptsfc_viz/plot_data/rankings.csv

 "model","n_eval_cases","mean_rk_wind","mean_rk_temp","mean_rk_dax","mean_rk_overall"
-"AryaStark",92,6.6,8.6,11.8,9
-"Bronn",92,14.6,11.6,14.2,13.4666666666667
-"CaptainRaymondHolt",92,7.2,5.6,6.8,6.53333333333333
-"ChandlerBing",92,8,17.8,10.2,12
-"DexterJettster",92,14.6,8.6,11.8,11.6666666666667
-"DougJudy",92,14,4,6.8,8.26666666666667
-"GeneralGrievous",92,20,19.2,19,19.4
-"GinaLinetti",92,20.8,16.6,14.4,17.2666666666667
-"HanSolo",92,7.2,15.2,9.6,10.6666666666667
-"HotPie",92,8,6.6,5.8,6.8
-"JabbaTheHutt",92,17.6,19.4,19.2,18.7333333333333
-"Joey",92,8.2,9.8,4.6,7.53333333333333
-"Joffrey_Baratheon",92,17.6,11.4,10.8,13.2666666666667
-"KyloRen",92,5.6,7.2,8.4,7.06666666666667
-"ObiWanKenobi",92,8.8,6.4,13.8,9.66666666666667
-"PhoebeBuffay",92,3.8,6.8,5.6,5.4
-"RossGeller",92,18.8,21,20.6,20.1333333333333
-"SamwellTarly",92,15,16.4,15.4,15.6
-"Shaggydog",92,3.6,7.2,9.6,6.8
-"UglyNakedGuy",92,6,7,8.2,7.06666666666667
-"Yoda",92,5,4.6,4.4,4.66666666666667
+"AryaStark",94,6.6,8.6,11.4,8.86666666666667
+"Bronn",94,14.6,11.6,13.6,13.2666666666667
+"CaptainRaymondHolt",94,7.2,5.6,5.8,6.2
+"ChandlerBing",94,8,17.8,9.8,11.8666666666667
+"DexterJettster",94,14.6,8.6,11,11.4
+"DougJudy",94,14,4,8.4,8.8
+"GeneralGrievous",94,20,19.2,18.8,19.3333333333333
+"GinaLinetti",94,20.8,16.6,15,17.4666666666667
+"HanSolo",94,7.2,15.2,8.4,10.2666666666667
+"HotPie",94,8,6.6,5.6,6.73333333333333
+"JabbaTheHutt",94,17.6,19.4,19.4,18.8
+"Joey",94,8.2,9.8,4.2,7.4
+"Joffrey_Baratheon",94,17.6,11.4,13.6,14.2
+"KyloRen",94,5.6,7.2,7.2,6.66666666666667
+"ObiWanKenobi",94,8.8,6.4,13.2,9.46666666666667
+"PhoebeBuffay",94,3.8,6.8,5.6,5.4
+"RossGeller",94,18.8,21,20.6,20.1333333333333
+"SamwellTarly",94,15,16.4,18.2,16.5333333333333
+"Shaggydog",94,3.6,7.2,9.4,6.73333333333333
+"UglyNakedGuy",94,6,7,7.8,6.93333333333333
+"Yoda",94,5,4.6,4,4.53333333333333

ptsfc_viz/plot_data/t2m.csv

 "time","value"
-2021-07-16 00:00:00,19.6
-2021-07-16 12:00:00,28.8
-2021-07-17 00:00:00,21.8
-2021-07-17 12:00:00,26.3
 2021-07-18 00:00:00,19
 2021-07-18 12:00:00,25.8
 2021-07-19 00:00:00,19.2
@@ -299,3 +295,7 @@
 2021-12-11 12:00:00,-0.3
 2021-12-12 00:00:00,-1.6
 2021-12-12 12:00:00,1.4
+2021-12-13 00:00:00,5.3
+2021-12-13 12:00:00,7.4
+2021-12-14 00:00:00,7.9
+2021-12-14 12:00:00,7.9

ptsfc_viz/plot_data/wind.csv

 "time","value"
-2021-07-14 12:00:00,12.96
-2021-07-15 00:00:00,7.56
-2021-07-15 12:00:00,7.56
-2021-07-16 00:00:00,2.52
 2021-07-16 12:00:00,11.16
 2021-07-17 00:00:00,8.64
 2021-07-17 12:00:00,18.72
@@ -299,3 +295,7 @@
 2021-12-11 12:00:00,6.84
 2021-12-12 00:00:00,10.44
 2021-12-12 12:00:00,7.92
+2021-12-13 00:00:00,10.08
+2021-12-13 12:00:00,5.76
+2021-12-14 00:00:00,7.92
+2021-12-14 12:00:00,10.08