diff --git a/NAMESPACE b/NAMESPACE index 793f89f..686f070 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,221 +1,219 @@ #exportPattern("^[[:alpha:]]+") export( "aggregate.bootstrap", "aggregate.bootstrap.list", "aggregate.challenge", "aggregate.ranked", "aggregate.ranked.list", "aggregateList", #? "aggregateThenRank", "as.challenge", "as.relation.challenge.incidence", "as.relation.ranked.list", "bootstrap", "bootstrap.ranked", "bootstrap.ranked.list", #"bootstrap.default", "boxplot.ranked.list", "boxplot.bootstrap.list", "boxplot.comparedRanks.list", #"check_strict_preference", "compareRanks", "compareRanks.ranked", "compareRanks.ranked.list", # "compareRanks.default", "consensus", "consensus.ranked.list", #"consensus.default", "Decision", "decision.challenge", "blobplot","blobplot.ranked", "default_colors", "density.bootstrap.list", "extract.workflow", "kendall", "kendall.bootstrap.list", # "melt.aggregated.list", "melt.ranked", "melt.ranked.list", # "merge.list", "lineplot.challenge", "methodsplot","methodsplot.challenge", "network", "plot.network", "podium", "podium.ranked","podium.ranked.list", "podium.challenge",#"podium.default", "print.aggregated", "print.aggregated.list", "print.comparedRanks", "print.ranked", "print.ranked.list", #"quickmerge.list", "rank", "rank.aggregated", "rank.aggregated.list", "rank.aggregatedRanks", "rank.aggregatedRanks.list", "rank.challenge", #"rank.default", "rankFrequencies", "rankFrequencies.bootstrap", "rankFrequencies.bootstrap.list", #"rankFrequencies.default", #"rankNA2", "rankThenAggregate", "rankingHeatmap", "rankingHeatmap.ranked.list", "relation_dissimilarity.ranked.list", "report", "report.bootstrap.list", "second", "select.if", "select.if.aggregated.list", "select.if.comparedRanks.list", "select.if.list", "select.if.ranked.list", "significanceMap", "spearmansFootrule", "spearmansWeightedFootrule", "splitby", "stability", "stabilityByAlgorithm", "stabilityByAlgorithmStacked","stabilityByTask", "stability.ranked.list", "stability.bootstrap","relation_dissimilarity", "stabilityByAlgorithm.bootstrap.list", "stabilityByAlgorithmStacked.bootstrap.list", "stabilityByTask.bootstrap.list", - "subset.aggregated.list", "subset.comparedRanks.list", "subset.list", "subset.ranked.list", "subset.bootstrap.list", "subset.bootstrap",#"which.top", + "subset.aggregated.list", "subset.comparedRanks.list", "subset.list", "subset.ranked.list", "subset.bootstrap.list", #"which.top", "test", "test.challenge", "test.default", "testThenRank", "violin", "violin.bootstrap.list", "winner", "winner.bootstrap", "winner.bootstrap.list", "winner.default", "winner.ranked", "winner.ranked.list", "winnerFrequencies", "winnerFrequencies.bootstrap", "winnerFrequencies.bootstrap.list", "winnerFrequencies.default" ) importFrom("dplyr", "bind_rows","group_by","summarise","select_if","filter","mutate","right_join","anti_join","ungroup","arrange","desc") importFrom("rlang",":=",".data","!!") importFrom("reshape2","melt", "acast") importFrom("utils", "capture.output", "methods") importFrom("plyr", "llply") importFrom("knitr", "kable") importFrom("tidyr", "complete","expand") importFrom("purrr", "%>%") importFrom("rmarkdown", "render","word_document","pdf_document","html_document") importFrom("viridisLite", "viridis","cividis") importFrom("ggplot2", "aes","aes_string","geom_abline", "geom_bar", "geom_boxplot", "geom_count", "geom_density", "geom_jitter", "geom_line", "geom_point", "geom_raster", "geom_step", "geom_text", "geom_violin","annotate","guide_legend", "geom_vline", "ggplot", "ggtitle","vars","xlab","ylab","scale_size_area","theme_get","rel","geom_hline","ggplot_build","scale_fill_manual", "scale_y_continuous","coord_cartesian", "element_text", "facet_wrap", "position_jitter", "stat", "stat_summary", "theme", "unit","guides","scale_fill_viridis_c") importFrom("grDevices", "col2rgb", "gray", "rgb", "grey") importFrom("graphics", "abline", "axis", "barplot", "box", "layout", "legend", "par", "plot", "points", "segments","boxplot", "stripchart", "title", "grconvertX", "plot.new") importFrom("stats", "as.dist", "as.formula", "median", "p.adjust", "density", "quantile", "aggregate", "cor", "wilcox.test", "terms.formula", "complete.cases") importFrom("methods", "new") importFrom("relations","relation","as.relation", "relation_domain", "relation_incidence", "relation_is_asymmetric","relation_consensus","relation_ensemble", "relation_is_irreflexive", "relation_is_negatively_transitive", "relation_is_transitive", "relation_is_trichotomous", "relation_scores", "relation_violations","relation_dissimilarity") importFrom("graph", "addEdge") S3method(print, comparedRanks) S3method(print, aggregated) S3method(print, ranked) S3method(print, aggregated.list) S3method(print, ranked.list) S3method(aggregate, challenge) S3method(aggregate, ranked) S3method(aggregate, ranked.list) S3method(aggregate, bootstrap.list) S3method(aggregate, bootstrap) S3method(test, default) S3method(test, challenge) S3method(Aggregate, default) S3method(Aggregate, data.frame) S3method(Aggregate, list) S3method(Rank, default) S3method(Rank, data.frame) S3method(Rank, list) S3method(rank, default) S3method(rank, challenge) S3method(rank, aggregated) S3method(rank, aggregated.list) S3method(rank, aggregatedRanks) S3method(rank, aggregatedRanks.list) S3method(blobplot, default) S3method(blobplot, ranked) S3method(bootstrap, default) S3method(bootstrap, ranked) S3method(bootstrap, ranked.list) S3method(winner, default) S3method(winner, ranked) S3method(winner, ranked.list) S3method(winner, bootstrap) S3method(winner, bootstrap.list) S3method(rankFrequencies, default) S3method(rankFrequencies, bootstrap) S3method(rankFrequencies, bootstrap.list) S3method(winnerFrequencies, default) S3method(winnerFrequencies, bootstrap) S3method(winnerFrequencies, bootstrap.list) S3method(compareRanks,default) S3method(compareRanks,ranked) S3method(compareRanks,ranked.list) S3method(merge,list) S3method(melt,ranked.list) S3method(melt,ranked) S3method(melt,aggregated.list) S3method(boxplot,ranked.list) S3method(boxplot,comparedRanks.list) S3method(boxplot,bootstrap.list) S3method(select.if,default) S3method(select.if,list) S3method(select.if,aggregated.list) S3method(select.if,ranked.list) S3method(select.if,comparedRanks.list) S3method(subset,list) S3method(subset,bootstrap.list) S3method(subset,aggregated.list) S3method(subset,ranked.list) S3method(subset,comparedRanks.list) -S3method(subset,bootstrap) - S3method(podium,default) S3method(podium,challenge) S3method(podium,ranked) S3method(podium,ranked.list) S3method(network,default) S3method(network,ranked.list) S3method(network,dist) S3method(plot,network) S3method(density,bootstrap.list) S3method(as.relation,challenge.incidence) S3method(as.relation,ranked.list) S3method(subset,bootstrap.list) S3method(subset,ranked.list) S3method(subset,list) S3method(subset,comparedRanks.list) S3method(subset,aggregated.list) S3method(decision,challenge) S3method(decision,default) S3method(lineplot,challenge) S3method(lineplot,default) S3method(methodsplot,challenge) S3method(methodsplot,default) S3method(significanceMap,data.frame) S3method(significanceMap,ranked.list) S3method(significanceMap,default) S3method(violin,bootstrap.list) S3method(violin,default) S3method(rankingHeatmap,ranked.list) S3method(rankingHeatmap,default) S3method(relation_dissimilarity,ranked.list) S3method(relation_dissimilarity,default) S3method(stabilityByTask,bootstrap.list) S3method(stabilityByTask,default) S3method(stability,bootstrap) S3method(stability,default) S3method(stability,ranked.list) S3method(stabilityByAlgorithm,bootstrap.list) S3method(stabilityByAlgorithm,default) S3method(stabilityByAlgorithmStacked,bootstrap.list) S3method(stabilityByAlgorithmStacked,default) S3method(consensus,ranked.list) S3method(consensus,default) S3method(report,bootstrap.list) S3method(report,ranked.list) S3method(report,default) diff --git a/R/subset.R b/R/subset.R index 238dfe4..463fc54 100644 --- a/R/subset.R +++ b/R/subset.R @@ -1,89 +1,73 @@ subset.comparedRanks.list=function(x, tasks,...){ res=x[tasks] class(res)="comparedRanks.list" res } subset.list=function(x, tasks,...){ x[tasks] } -subset.bootstrap.list=function(x, - tasks,...){ - if (!is.null(as.list(match.call(expand.dots = T))$top)) stop("Subset of algorithms only sensible for single task challenges.") - res=list(bootsrappedRanks=x$bootsrappedRanks[tasks], - bootsrappedAggregate=x$bootsrappedAggregate[tasks], - matlist=x$matlist[tasks], - data=x$data[tasks], - FUN=x$FUN - ) - - attrib=attributes(x$data) - attrib$names=attr(res$data,"names") - attributes(res$data)=attrib - class(res)="bootstrap.list" - res - -} - subset.aggregated.list=function(x, tasks,...){ call=match.call(expand.dots = T) if (!is.null(as.list(call$top))) stop("Subset of algorithms only sensible for single task challenges.") matlist=x$matlist[tasks] res=list(matlist=matlist, call=list(x$call,call), data=x$data, FUN = . %>% (x$FUN) %>% (call) ) class(res)=class(x) res } which.top=function(object, top){ mat=object$mat[object$mat$rank<=top,] rownames(mat)#[order(mat$rank)] } subset.ranked.list=function(x, top,...) { if (length(x$matlist) != 1) { stop("Subset of algorithms only sensible for single-task challenges.") } taskMat <- x$matlist[[1]] taskData <- x$data[[1]] objectTop=x objectTop$matlist[[1]]=taskMat[taskMat$rank<=top,] taskMatRowNames <- rownames(objectTop$matlist[[1]]) attribute <- attr(objectTop$data,"algorithm") selectedRowNames <- taskData[[attribute]] %in% taskMatRowNames objectTop$data[[1]] <- taskData[selectedRowNames,] objectTop$data[[1]][[attribute]] <- droplevels(objectTop$data[[1]][[attribute]]) objectTop$fulldata=x$data objectTop } -subset.bootstrap=function(x, - top,...){ - objectTop=x - objectTop$mat=objectTop$mat[objectTop$mat$rank<=top,] - objectTop$data=objectTop$data[objectTop$data[[attr(objectTop$data,"algorithm")]]%in% rownames(objectTop$mat),] - objectTop$data[[attr(objectTop$data,"algorithm")]]=droplevels(objectTop$data[[attr(objectTop$data,"algorithm")]]) - objectTop$fulldata=x$data - objectTop$bootsrappedRanks=objectTop$bootsrappedRanks[rownames(objectTop$mat),] - objectTop$bootsrappedAggregate=objectTop$bootsrappedAggregate[rownames(objectTop$mat),] +subset.bootstrap.list=function(x, + top,...) { + + if (length(x$matlist) != 1) { + stop("Subset of algorithms only sensible for single-task challenges.") + } + + objectTop <- subset.ranked.list(x, top = top) + + objectTop$bootsrappedRanks[[1]] <- objectTop$bootsrappedRanks[[1]][rownames(objectTop$matlist[[1]]),] + objectTop$bootsrappedAggregate[[1]] <- objectTop$bootsrappedAggregate[[1]][rownames(objectTop$matlist[[1]]),] objectTop } diff --git a/tests/testthat/test-subset.R b/tests/testthat/test-subset.R index d08316c..54822a1 100644 --- a/tests/testthat/test-subset.R +++ b/tests/testthat/test-subset.R @@ -1,102 +1,135 @@ test_that("top 2 performing algorithms are extracted and data set is reduced respectively", { data <- rbind( data.frame(algo="A1", value=0.8, case="C1"), data.frame(algo="A2", value=0.6, case="C1"), data.frame(algo="A3", value=0.4, case="C1"), data.frame(algo="A1", value=0.2, case="C2"), data.frame(algo="A2", value=0.1, case="C2"), data.frame(algo="A3", value=0.0, case="C2")) challenge <- as.challenge(data, taskName="T1", algorithm="algo", case="case", value="value", smallBetter=FALSE) ranking <- challenge%>%aggregateThenRank(FUN=mean, ties.method="min") rankingSubset <- subset(ranking, top=2) expectedRankingSubset <- rbind( "A1" = data.frame(value_FUN = 0.5, rank = 1), "A2" = data.frame(value_FUN = 0.35, rank = 2)) expect_equal(rankingSubset$matlist$T1, expectedRankingSubset) expect_equal(as.vector(rankingSubset$data$T1$algo), c("A1", "A2", "A1", "A2")) expect_equal(as.vector(rankingSubset$data$T1$value), c(0.8, 0.6, 0.2, 0.1)) expect_equal(as.vector(rankingSubset$data$T1$case), c("C1", "C1", "C2", "C2")) expect_equal(as.vector(rankingSubset$data$T1$task), c("T1", "T1", "T1", "T1")) }) test_that("extraction of subset raises error for multi-task data set", { dataTask1 <- cbind(task="T1", rbind( data.frame(algo="A1", value=0.8, case="C1"), data.frame(algo="A2", value=0.6, case="C1"), data.frame(algo="A3", value=0.4, case="C1"), data.frame(algo="A1", value=0.2, case="C2"), data.frame(algo="A2", value=0.1, case="C2"), data.frame(algo="A3", value=0.0, case="C2") )) dataTask2 <- cbind(task="T2", rbind( data.frame(algo="A1", value=0.2, case="C1"), data.frame(algo="A2", value=0.3, case="C1"), data.frame(algo="A3", value=0.4, case="C1"), data.frame(algo="A1", value=0.7, case="C2"), data.frame(algo="A2", value=0.8, case="C2"), data.frame(algo="A3", value=0.9, case="C2") )) data <- rbind(dataTask1, dataTask2) challenge <- as.challenge(data, by="task", algorithm="algo", case="case", value="value", smallBetter=FALSE) ranking <- challenge%>%aggregateThenRank(FUN=mean, ties.method="min") expect_error(subset(ranking, top=2), "Subset of algorithms only sensible for single-task challenges.", fixed=TRUE) }) test_that("extraction of subset returns all algorithms even when more are requested", { data <- rbind( data.frame(algo="A1", value=0.8, case="C1"), data.frame(algo="A2", value=0.6, case="C1"), data.frame(algo="A3", value=0.4, case="C1"), data.frame(algo="A1", value=0.2, case="C2"), data.frame(algo="A2", value=0.1, case="C2"), data.frame(algo="A3", value=0.0, case="C2")) challenge <- as.challenge(data, taskName="T1", algorithm="algo", case="case", value="value", smallBetter=FALSE) ranking <- challenge%>%aggregateThenRank(FUN=mean, ties.method="min") rankingSubset <- subset(ranking, top=4) expectedRankingSubset <- rbind( "A1" = data.frame(value_FUN = 0.5, rank = 1), "A2" = data.frame(value_FUN = 0.35, rank = 2), "A3" = data.frame(value_FUN = 0.2, rank = 3)) expect_equal(rankingSubset$matlist$T1, expectedRankingSubset) }) test_that("extraction of subset returns more algorithms then requested when ties are present", { data <- rbind( data.frame(algo="A1", value=0.8, case="C1"), data.frame(algo="A2", value=0.8, case="C1"), data.frame(algo="A3", value=0.8, case="C1"), data.frame(algo="A1", value=0.2, case="C2"), data.frame(algo="A2", value=0.2, case="C2"), data.frame(algo="A3", value=0.2, case="C2")) challenge <- as.challenge(data, taskName="T1", algorithm="algo", case="case", value="value", smallBetter=FALSE) ranking <- challenge%>%aggregateThenRank(FUN=mean, ties.method="min") rankingSubset <- subset(ranking, top=2) expectedRankingSubset <- rbind( "A1" = data.frame(value_FUN = 0.5, rank = 1), "A2" = data.frame(value_FUN = 0.5, rank = 1), "A3" = data.frame(value_FUN = 0.5, rank = 1)) expect_equal(rankingSubset$matlist$T1, expectedRankingSubset) }) + +test_that("top 2 performing algorithms are extracted from bootstrap ranking and data set is reduced respectively", { + data <- rbind( + data.frame(algo="A1", value=0.8, case="C1"), + data.frame(algo="A2", value=0.6, case="C1"), + data.frame(algo="A3", value=0.4, case="C1"), + data.frame(algo="A1", value=0.2, case="C2"), + data.frame(algo="A2", value=0.1, case="C2"), + data.frame(algo="A3", value=0.0, case="C2")) + + challenge <- as.challenge(data, taskName="T1", algorithm="algo", case="case", value="value", smallBetter=FALSE) + + ranking <- challenge%>%aggregateThenRank(FUN=mean, ties.method="min") + + set.seed(1) + rankingBootstrapped <- ranking%>%bootstrap(nboot=10) + + rankingBootstrappedSubset <- subset(rankingBootstrapped, top=2) + + expectedRankingSubset <- rbind( + "A1" = data.frame(value_FUN = 0.5, rank = 1), + "A2" = data.frame(value_FUN = 0.35, rank = 2)) + + expect_equal(rankingBootstrappedSubset$matlist$T1, expectedRankingSubset) + + expect_equal(as.vector(rankingBootstrappedSubset$data$T1$algo), c("A1", "A2", "A1", "A2")) + expect_equal(as.vector(rankingBootstrappedSubset$data$T1$value), c(0.8, 0.6, 0.2, 0.1)) + expect_equal(as.vector(rankingBootstrappedSubset$data$T1$case), c("C1", "C1", "C2", "C2")) + expect_equal(as.vector(rankingBootstrappedSubset$data$T1$task), c("T1", "T1", "T1", "T1")) + + expect_equal(dim(rankingBootstrappedSubset$bootsrappedRanks$T1), c(2, 10)) + expect_equal(dim(rankingBootstrappedSubset$bootsrappedAggregate$T1), c(2, 10)) +})