diff --git a/msd-rankings-subtasks-paper.R b/msd-rankings-subtasks-paper.R index 2285f88..7f24b3a 100644 --- a/msd-rankings-subtasks-paper.R +++ b/msd-rankings-subtasks-paper.R @@ -1,120 +1,148 @@ library(rstudioapi) current_path = rstudioapi::getActiveDocumentContext()$path setwd(dirname(current_path)) library(challengeR) library(doParallel) library(plyr) source("msd-prepare-data.R") ########################### # Define functions ########################### computeSubtaskResults <- function(data) { # Create challenge object challenge= data %>% - as.challenge(value = "value", algorithm = "alg_name",case = "case_task",by = "subtask", smallBetter = F, na.treat = 0) + as.challenge(value = "value", algorithm = "alg_name",case = "case_task", by = "subtask", smallBetter = F, na.treat = 0) # Compute significance ranking sigRanking=challenge %>% testThenRank(ties.method = "min",alpha=.05) # Compute consensus ranking over all subtasks (regions) consensusRanks <- sigRanking%>%consensus(method = "euclidean") # Get subtask rankings algorithms <- names(consensusRanks) subtasks <- unique(data$subtask) rankings <- getSubtaskRankings(sigRanking, subtasks, algorithms) return(rankings) } getSubtaskRankings <- function(data, subtasks, sorting) { # Create data frame for rankings rankings <- data.frame(algorithm = sorting) colnames(rankings) <- c("Algorithm") for(subtask in subtasks) { # Select subtask rankings from ranking lists subtaskRanking <- tibble::rownames_to_column(data$matlist[subtask][[1]], "Algorithm") sigRanking <- subtaskRanking[match(sorting, subtaskRanking$Algorithm),] new_colnames <- c(colnames(rankings), paste("rank",subtask, sep="_")) # Write all rankings in one data frame rankings <- cbind(rankings, sigRanking$rank) colnames(rankings) <- new_colnames } return(rankings) } +getAggregatedRanking <- function(data, rankings) { + # Initialize data frame for all organ average ranks + rankingsAllOrgans <- data.frame(Algorithm = rankings$Algorithm) + + tasks <- unique(data$task) + + # Iterate over all tasks + for(t in tasks) { + # Get corresponding subtasks + subtasks <- unique(subset(data, task == t)$subtask) + + rankingsOrgan <- data.frame(matrix(vector(), 19, 0)) + # Iterate over subtasks + for(s in subtasks) { + rankingsOrgan <- cbind(rankingsOrgan, rankings[, grepl(s, names(rankings))]) + } + + # Average ranks + rankingsOrgan$meanRank <- rowMeans(rankingsOrgan) + + # Rank over averages + rankingsOrgan$rank <- rank(rankingsOrgan$meanRank) + + # Add to data frame + rankingsAllOrgans[,ncol(rankingsAllOrgans)+1] <- rankingsOrgan$rank + colnames(rankingsAllOrgans)[ncol(rankingsAllOrgans)] <- paste("rank", t, sep="_") + } + + rankingsAllOrgans$MeanRank <- rowMeans(rankingsAllOrgans[,2:ncol(rankingsAllOrgans)]) + rankingsAllOrgans$rank <- rank(rankingsAllOrgans$MeanRank) + + return(rankingsAllOrgans) +} + ########################### # Read and prepare challenge data ########################### msd_data_matrix <- prepare_data() # Prepare data subsets for each metric and phase dataDSC= msd_data_matrix %>% subset(score=="DSC") dataNSD= msd_data_matrix %>% subset(score=="NSD") dataDSCph1= msd_data_matrix %>% subset(score=="DSC" & phase==1) dataNSDph1= msd_data_matrix %>% subset(score=="NSD" & phase==1) dataDSCph2= msd_data_matrix %>% subset(score=="DSC" & phase==2) dataNSDph2= msd_data_matrix %>% subset(score=="NSD" & phase==2) ########################### # Compute Phase 1 Results ########################### ## DSC metric results rankingsPh1_dsc <- computeSubtaskResults(dataDSCph1) write.csv(rankingsPh1_dsc, "rankings-per-subtask/subtask-rankings-ph1-dsc.csv") ## NSD metric results rankingsPh1_nsd <- computeSubtaskResults(dataNSDph1) write.csv(rankingsPh1_nsd, "rankings-per-subtask/subtask-rankings-ph1-nsd.csv") -## Aggregate rankings by averaging over all subtasks -rankingsMeanPh1 <- cbind(rankingsPh1_dsc, rankingsPh1_nsd[,2:14]) +# Merge DSC and NSD rankings in one data frame and rename columns +rankingsPh1 <- cbind(rankingsPh1_dsc, rankingsPh1_nsd[,2:14]) +colnames(rankingsPh1)[2:14] <- paste(colnames(rankingsPh1)[2:14], "dsc", sep="_") +colnames(rankingsPh1)[15:27] <- paste(colnames(rankingsPh1)[15:27], "nsd", sep="_") -# Calculate average ranks -rankingsMeanPh1 <- cbind(rankingsMeanPh1,rowMeans(rankingsMeanPh1[,2:27])) +# Get aggregated ranking +aggregatedRankingPh1 <- getAggregatedRanking(dataDSCph1, rankingsPh1) -# Remove individual subtask ranks, only keep mean -rankingsMeanPh1[,2:27] <- NULL -colnames(rankingsMeanPh1) <- c("Algorithm", "MeanRank") +write.csv(aggregatedRankingPh1, "rankings-per-subtask/mean-ranking-ph1.csv") -# Rank average rankings -rankingsMeanPh1$rank <- rank(rankingsMeanPh1$MeanRank) -write.csv(rankingsMeanPh1, "rankings-per-subtask/mean-ranking-ph1.csv") ########################### # Compute Phase 2 Results ########################### ## DSC metric results rankingsPh2_dsc <- computeSubtaskResults(dataDSCph2) write.csv(rankingsPh2_dsc, "rankings-per-subtask/subtask-rankings-ph2-dsc.csv") ## NSD metric results rankingsPh2_nsd <- computeSubtaskResults(dataNSDph2) write.csv(rankingsPh2_nsd, "rankings-per-subtask/subtask-rankings-ph2-nsd.csv") -## Aggregate rankings by averaging over all subtasks -rankingsMeanPh2 <- cbind(rankingsPh2_dsc, rankingsPh2_nsd[,2:5]) - -# Calculate average ranks -rankingsMeanPh2 <- cbind(rankingsMeanPh2,rowMeans(rankingsMeanPh2[,2:9])) +## Aggregate rankings: Average per organ, then average over organ-scores +# Merge DSC and NSD rankings in one data frame and rename columns +rankingsPh2 <- cbind(rankingsPh2_dsc, rankingsPh2_nsd[,2:5]) +colnames(rankingsPh2)[2:5] <- paste(colnames(rankingsPh2)[2:5], "dsc", sep="_") +colnames(rankingsPh2)[6:9] <- paste(colnames(rankingsPh2)[6:9], "nsd", sep="_") -# Remove individual subtask ranks, only keep mean -rankingsMeanPh2[,2:9] <- NULL -colnames(rankingsMeanPh2) <- c("Algorithm", "MeanRank") +# Get aggregated ranking +aggregatedRankingPh2 <- getAggregatedRanking(dataDSCph2, rankingsPh2) -# Rank average rankings -rankingsMeanPh2$rank <- rank(rankingsMeanPh2$MeanRank) -write.csv(rankingsMeanPh2, "rankings-per-subtask/mean-ranking-ph2.csv") +write.csv(aggregatedRankingPh2, "rankings-per-subtask/mean-ranking-ph2.csv") diff --git a/rankings-per-subtask/mean-ranking-ph1.csv b/rankings-per-subtask/mean-ranking-ph1.csv index 82eece0..3fce5e2 100644 --- a/rankings-per-subtask/mean-ranking-ph1.csv +++ b/rankings-per-subtask/mean-ranking-ph1.csv @@ -1,20 +1,20 @@ -"","Algorithm","MeanRank","rank" -"1","nnU-Net",1.34615384615385,1 -"2","K.A.V.athlon",2.96153846153846,2 -"3","NVDLMED",4.46153846153846,3 -"4","CerebriuDIKU",4.57692307692308,4 -"5","lupin",4.80769230769231,5 -"6","LS Wang's Group",5.30769230769231,6 -"7","MIMI",6.96153846153846,7 -"8","whale",8.88461538461539,8 -"9","BCVuniandes",9.19230769230769,9 -"10","VST",9.88461538461539,10 -"11","UBIlearn",10.3076923076923,11 -"12","BUT",10.5,12 -"13","A-REUMI01",10.6153846153846,13 -"14","LfB",11.8846153846154,14 -"15","jiafucang",13.3461538461538,15 -"16","AI-MED",14.4615384615385,16 -"17","Lesswire1",15.3461538461538,17 -"18","RegionTec",16.3846153846154,18 -"19","EdwardMa12593",16.7307692307692,19 +"","Algorithm","rank_Brain","rank_Heart","rank_Liver","rank_Hippocampus","rank_Prostate","rank_Lung","rank_Pancreas","MeanRank","rank" +"1","nnU-Net",1,1.5,1,1,1,1,1,1.07142857142857,1 +"2","K.A.V.athlon",3,4,3,2,3.5,2,3,2.92857142857143,2 +"3","NVDLMED",4,1.5,2,8,6,8,2,4.5,3 +"4","CerebriuDIKU",2,10.5,7,3.5,5,3.5,5,5.21428571428571,5 +"5","lupin",7,3,4.5,3.5,3.5,6,6.5,4.85714285714286,4 +"6","LS Wang's Group",5,7.5,9,5,8.5,3.5,4,6.07142857142857,6 +"7","MIMI",8,12.5,8,6,7,5,9.5,8,7 +"8","whale",16,5,6,13,2,10,11,9,8 +"9","BCVuniandes",6,12.5,13.5,7,8.5,14.5,16,11.1428571428571,12 +"10","VST",12,7.5,11,9,11,10,9.5,10,9 +"11","UBIlearn",14,7.5,10,11.5,13,10,6.5,10.3571428571429,10 +"12","BUT",9,15,4.5,16,15,14.5,8,11.7142857142857,13 +"13","A-REUMI01",10,7.5,13.5,11.5,10,13,12,11.0714285714286,11 +"14","LfB",15,10.5,16,14,12,7,14,12.6428571428571,14 +"15","jiafucang",18,14,18,15,14,12,13,14.8571428571429,15 +"16","AI-MED",11,16,12,17,16,16.5,17.5,15.1428571428571,16 +"17","Lesswire1",13,18,15,18,18,16.5,17.5,16.5714285714286,17 +"18","RegionTec",17,18,17,19,19,19,15,17.7142857142857,19 +"19","EdwardMa12593",19,18,19,10,17,18,19,17.1428571428571,18 diff --git a/rankings-per-subtask/mean-ranking-ph2.csv b/rankings-per-subtask/mean-ranking-ph2.csv index 3910618..e7f11c3 100644 --- a/rankings-per-subtask/mean-ranking-ph2.csv +++ b/rankings-per-subtask/mean-ranking-ph2.csv @@ -1,20 +1,20 @@ -"","Algorithm","MeanRank","rank" -"1","nnU-Net",1,1 -"2","NVDLMED",1.625,2 -"3","K.A.V.athlon",2.25,3 -"4","LS Wang's Group",4.375,4 -"5","MIMI",6,5 -"6","CerebriuDIKU",6.5,6 -"7","whale",7.125,7 -"8","UBIlearn",7.875,8 -"9","lupin",9,9 -"10","jiafucang",9.75,10 -"11","A-REUMI01",10.25,11 -"12","LfB",10.375,12 -"13","VST",12.375,13 -"14","BUT",13.375,14 -"15","AI-MED",13.75,15 -"16","Lesswire1",14,16 -"17","RegionTec",15.5,17 -"18","BCVuniandes",16.375,18 -"19","EdwardMa12593",17.25,19 +"","Algorithm","rank_Hepatic Vessel","rank_Spleen","rank_Colon","MeanRank","rank" +"1","nnU-Net",1,1,1.5,1.16666666666667,1 +"2","NVDLMED",2,3,1.5,2.16666666666667,2 +"3","K.A.V.athlon",3,2,3.5,2.83333333333333,3 +"4","LS Wang's Group",5.5,4,3.5,4.33333333333333,4 +"5","MIMI",4,7,6.5,5.83333333333333,5 +"6","CerebriuDIKU",7,8,5,6.66666666666667,6 +"7","whale",8,6,6.5,6.83333333333333,7 +"8","UBIlearn",9,5,12,8.66666666666667,8 +"9","lupin",5.5,10,16,10.5,10 +"10","jiafucang",11,9,11,10.3333333333333,9 +"11","A-REUMI01",10,16,8.5,11.5,12 +"12","LfB",12,12,8.5,10.8333333333333,11 +"13","VST",14,11,14,13,13 +"14","BUT",13,14.5,18.5,15.3333333333333,15.5 +"15","AI-MED",16,14.5,10,13.5,14 +"16","Lesswire1",15,18,13,15.3333333333333,15.5 +"17","RegionTec",18,13,16,15.6666666666667,17 +"18","BCVuniandes",17,17,18.5,17.5,18 +"19","EdwardMa12593",19,19,16,18,19