diff --git a/.gitignore b/.gitignore index f46e7dd..784180b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,106 +1,108 @@ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # latex *.aux *.bbl *.blg *.log *.tcp # solver_comparison -examples/solver_comparison/gfx/* +examples/solver_comparison/gfx/data_I +examples/solver_comparison/gfx/data_II +examples/solver_comparison/gfx/data_III *.vpp.lck .pytest_cache/ *.vpp.bak_* python_tests_xml # C extensions *.so # Distribution / packaging .Python env/ build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ *.egg-info/ .installed.cfg *.egg .idea/ # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *,cover .hypothesis/ # Translations *.mo *.pot # Django stuff: *.log # Sphinx documentation docs/_build/ doc/ # PyBuilder target/ #Ipython Notebook .ipynb_checkpoints #Pycharm files *.iml # merging stuff *.orig *~ # Paths in repository mcml.py # images etc *.tif *.nrrd *.caffemodel # C++ stuff build* *.user hyppopy/tests/test_snipped_000.py hyppopy/tests/test_snipped_001.py hyppopy/tests/test_snipped_002.py hyppopy/tests/test_snipped_003.py hyppopy/tests/test_snipped_004.py hyppopy/tests/test_snipped_005.py hyppopy/tests/test_snipped_006.py diff --git a/examples/solver_comparison.py b/examples/solver_comparison.py index 670f873..f4743b0 100644 --- a/examples/solver_comparison.py +++ b/examples/solver_comparison.py @@ -1,325 +1,364 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE import os import sys import time import pickle import numpy as np from math import pi import matplotlib.pyplot as plt from hyppopy.SolverPool import SolverPool from hyppopy.HyppopyProject import HyppopyProject from hyppopy.VirtualFunction import VirtualFunction from hyppopy.BlackboxFunction import BlackboxFunction #OUTPUTDIR = "C:\\Users\\s635r\\Desktop\\solver_comparison" OUTPUTDIR = "D:\\Projects\\Python\\hyppopy\\examples\\solver_comparison\\gfx" SOLVER = [] -#SOLVER.append("hyperopt") -#SOLVER.append("optunity") -#SOLVER.append("randomsearch") -#SOLVER.append("optuna") + SOLVER.append("quasirandomsearch") +SOLVER.append("randomsearch") +SOLVER.append("hyperopt") +SOLVER.append("optunity") +SOLVER.append("optuna") ITERATIONS = [] +ITERATIONS.append(15) ITERATIONS.append(50) -ITERATIONS.append(100) -ITERATIONS.append(250) -ITERATIONS.append(500) +ITERATIONS.append(300) +ITERATIONS.append(1000) -STATREPEATS = 1 +STATREPEATS = 50 OVERWRITE = False def compute_deviation(solver_name, vfunc_id, iterations, N, fname): project = HyppopyProject() project.add_hyperparameter(name="axis_00", domain="uniform", data=[0, 1], type=float) project.add_hyperparameter(name="axis_01", domain="uniform", data=[0, 1], type=float) project.add_hyperparameter(name="axis_02", domain="uniform", data=[0, 1], type=float) project.add_hyperparameter(name="axis_03", domain="uniform", data=[0, 1], type=float) project.add_hyperparameter(name="axis_04", domain="uniform", data=[0, 1], type=float) vfunc = VirtualFunction() vfunc.load_default(vfunc_id) minima = vfunc.minima() def my_loss_function(data, params): return vfunc(**params) blackbox = BlackboxFunction(data=[], blackbox_func=my_loss_function) - results = {} results["gt"] = [] for mini in minima: results["gt"].append(np.median(mini[0])) for iter in iterations: results[iter] = {"minima": {}, "distance": {}, "duration": None, "set_difference": None, "loss": None, "loss_history": {}} for i in range(vfunc.dims()): results[iter]["minima"]["axis_0{}".format(i)] = [] results[iter]["distance"]["axis_0{}".format(i)] = [] project.add_setting("max_iterations", iter) project.add_setting("solver", solver_name) solver = SolverPool.get(project=project) solver.blackbox = blackbox axis_minima = [] best_losses = [] best_sets_diff = [] for i in range(vfunc.dims()): axis_minima.append([]) loss_history = [] durations = [] for n in range(N): print("\rSolver={} iteration={} round={}".format(solver, iter, n), end="") start = time.time() solver.run(print_stats=False) end = time.time() durations.append(end-start) df, best = solver.get_results() loss_history.append(np.flip(np.sort(df['losses'].values))) best_row = df['losses'].idxmin() best_losses.append(df['losses'][best_row]) best_sets_diff.append(abs(df['axis_00'][best_row] - best['axis_00'])+ abs(df['axis_01'][best_row] - best['axis_01'])+ abs(df['axis_02'][best_row] - best['axis_02'])+ abs(df['axis_03'][best_row] - best['axis_03'])+ abs(df['axis_04'][best_row] - best['axis_04'])) for i in range(vfunc.dims()): tmp = df['axis_0{}'.format(i)][best_row] axis_minima[i].append(tmp) results[iter]["loss_history"] = loss_history for i in range(vfunc.dims()): results[iter]["minima"]["axis_0{}".format(i)] = [np.mean(axis_minima[i]), np.std(axis_minima[i])] dist = np.sqrt((axis_minima[i]-results["gt"][i])**2) results[iter]["distance"]["axis_0{}".format(i)] = [np.mean(dist), np.std(dist)] results[iter]["loss"] = [np.mean(best_losses), np.std(best_losses)] results[iter]["set_difference"] = sum(best_sets_diff) results[iter]["duration"] = np.mean(durations) file = open(fname, 'wb') pickle.dump(results, file) file.close() def make_radarplot(results, title, fname=None): gt = results.pop("gt") categories = list(results[list(results.keys())[0]]["minima"].keys()) N = len(categories) angles = [n / float(N) * 2 * pi for n in range(N)] angles += angles[:1] ax = plt.subplot(1, 1, 1, polar=True, ) ax.set_theta_offset(pi / 2) ax.set_theta_direction(-1) plt.xticks(angles[:-1], categories, color='grey', size=8) ax.set_rlabel_position(0) plt.yticks([0.2, 0.4, 0.6, 0.8, 1.0], ["0.2", "0.4", "0.6", "0.8", "1.0"], color="grey", size=7) plt.ylim(0, 1) gt += gt[:1] ax.fill(angles, gt, color=(0.2, 0.8, 0.2), alpha=0.2) colors = [] cm = plt.get_cmap('Set1') if len(results) > 2: indices = list(range(0, len(results) + 1)) indices.pop(2) else: indices = list(range(0, len(results))) for i in range(len(results)): colors.append(cm(indices[i])) for iter, data in results.items(): values = [] for i in range(len(categories)): values.append(data["minima"]["axis_0{}".format(i)][0]) values += values[:1] color = colors.pop(0) ax.plot(angles, values, color=color, linewidth=2, linestyle='solid', label="iterations {}".format(iter)) plt.title(title, size=11, color=(0.1, 0.1, 0.1), y=1.1) plt.legend(bbox_to_anchor=(0.08, 1.12)) if fname is None: plt.show() else: plt.savefig(fname + ".png") #plt.savefig(fname + ".svg") plt.clf() def make_errrorbars_plot(results, fname=None): n_groups = len(results) for iter in ITERATIONS: means = [] stds = [] names = [] colors = [] axis = [] fig = plt.figure(figsize=(10, 8)) for solver_name, numbers in results.items(): names.append(solver_name) means.append([]) stds.append([]) for axis_name, data in numbers[iter]["distance"].items(): means[-1].append(data[0]) stds[-1].append(data[1]) if len(axis) < 5: axis.append(axis_name) for c in range(len(names)): colors.append(plt.cm.Set2(c/len(names))) index = np.arange(len(axis)) bar_width = 0.14 opacity = 0.8 error_config = {'ecolor': '0.3'} for k, name in enumerate(names): plt.bar(index + k*bar_width, means[k], bar_width, alpha=opacity, color=colors[k], yerr=stds[k], error_kw=error_config, label=name) plt.xlabel('Axis') plt.ylabel('Mean [+/- std]') plt.title('Deviation per Axis and Solver for {} Iterations'.format(iter)) plt.xticks(index + 2*bar_width, axis) plt.legend() if fname is None: plt.show() else: plt.savefig(fname + "_{}.png".format(iter)) #plt.savefig(fname + "_{}.svg".format(iter)) plt.clf() def plot_loss_histories(results, fname=None): colors = [] for c in range(len(SOLVER)): colors.append(plt.cm.Set2(c / len(SOLVER))) for iter in ITERATIONS: fig = plt.figure(figsize=(10, 8)) added_solver = [] for n, solver_name in enumerate(results.keys()): for history in results[solver_name][iter]["loss_history"]: if solver_name not in added_solver: plt.plot(history, color=colors[n], label=solver_name, alpha=0.5) added_solver.append(solver_name) else: plt.plot(history, color=colors[n], alpha=0.5) plt.legend() plt.ylabel('Loss') plt.xlabel('Iteration') if fname is None: plt.show() else: plt.savefig(fname + "_{}.png".format(iter)) plt.clf() def print_durations(results, fname=None): - colors = [] - for c in range(len(SOLVER)): - colors.append(plt.cm.Set2(c / len(SOLVER))) - - f = open(fname, "w") - lines = ["\t".join(SOLVER)+"\n"] + # colors = [] + # for c in range(len(SOLVER)): + # colors.append(plt.cm.Set2(c / len(SOLVER))) + f = open(fname + ".txt", "w") + lines = ["iterations\t"+"\t".join(SOLVER)+"\n"] for iter in ITERATIONS: txt = str(iter) + "\t" for solver_name in SOLVER: duration = results[solver_name][iter]["duration"] txt += str(duration) + "\t" txt += "\n" lines.append(txt) f.writelines(lines) f.close() + durations = {} + for iter in ITERATIONS: + for solver_name in SOLVER: + duration = results[solver_name][iter]["duration"] + if not solver_name in durations: + durations[solver_name] = duration/iter + else: + durations[solver_name] += duration/iter + + for name in durations.keys(): + durations[name] /= len(ITERATIONS) + + fig, ax = plt.subplots(figsize=(14, 6)) + + # Example data + y_pos = np.arange(len(durations.keys())) -id2dirmapping = {"5D": "data_I", "5D2": "data_II", "5D3": "data_II"} + t = [] + for solver in SOLVER: + t.append(durations[solver]) + print(SOLVER) + print(t) + + ax.barh(y_pos, t, align='center', color='green') + ax.set_yticks(y_pos) + ax.set_yticklabels(SOLVER) + ax.invert_yaxis() + ax.set_xscale('log') + ax.set_xlabel('Duration in [s]') + ax.set_title('Mean Solver Computation Time per Iteration') + + if fname is None: + plt.show() + else: + plt.savefig(fname + ".png") + # plt.savefig(fname + "_{}.svg".format(iter)) + plt.clf() + + + +id2dirmapping = {"5D": "data_I", "5D2": "data_II", "5D3": "data_III"} if __name__ == "__main__": vfunc_ID = "5D" if len(sys.argv) == 2: vfunc_ID = sys.argv[1] print("Start Evaluation on {}".format(vfunc_ID)) OUTPUTDIR = os.path.join(OUTPUTDIR, id2dirmapping[vfunc_ID]) if not os.path.isdir(OUTPUTDIR): os.makedirs(OUTPUTDIR) ################################################## ############### create datasets ################## fnames = [] for solver_name in SOLVER: fname = os.path.join(OUTPUTDIR, solver_name) fnames.append(fname) if OVERWRITE or not os.path.isfile(fname): compute_deviation(solver_name, vfunc_ID, ITERATIONS, N=STATREPEATS, fname=fname) ################################################## ################################################## ################################################## ############## create radarplots ################# all_results = {} for solver_name, fname in zip(SOLVER, fnames): file = open(fname, 'rb') results = pickle.load(file) file.close() make_radarplot(results, solver_name, fname + "_deviation") all_results[solver_name] = results fname = os.path.join(OUTPUTDIR, "errorbars") make_errrorbars_plot(all_results, fname) fname = os.path.join(OUTPUTDIR, "losshistory") plot_loss_histories(all_results, fname) - fname = os.path.join(OUTPUTDIR, "durations.txt") + fname = os.path.join(OUTPUTDIR, "durations") print_durations(all_results, fname) for solver_name, iterations in all_results.items(): for iter, numbers in iterations.items(): if numbers["set_difference"] != 0: print("solver {} has a different parameter set match in iteration {}".format(solver_name, iter)) - ################################################## ################################################## + + plt.imsave(fname=os.path.join(OUTPUTDIR, "dummy.png"), arr=np.ones((800, 1000, 3), dtype=np.uint8)*255) diff --git a/examples/solver_comparison/HyppopyReport.pdf b/examples/solver_comparison/HyppopyReport.pdf index 6aafecb..59f1af9 100644 Binary files a/examples/solver_comparison/HyppopyReport.pdf and b/examples/solver_comparison/HyppopyReport.pdf differ diff --git a/examples/solver_comparison/HyppopyReport.tex b/examples/solver_comparison/HyppopyReport.tex index bbd194d..3d885f5 100644 --- a/examples/solver_comparison/HyppopyReport.tex +++ b/examples/solver_comparison/HyppopyReport.tex @@ -1,31 +1,35 @@ \title{Hyppopy Solver Comparison Report} \date{\today} +\def \myiterI {15} +\def \myiterII {50} +\def \myiterIII {300} +\def \myiterIV {1000} \documentclass[12pt]{article} \usepackage{geometry} \geometry{ a4paper, total={170mm,257mm}, left=20mm, top=20mm, } \usepackage{subcaption} \usepackage{graphicx} \graphicspath{{./gfx/}} \captionsetup[subfigure]{labelformat=empty} \begin{document} \maketitle \newpage \section{Benchmarking} \input{ReportPage_I.tex} \newpage \input{ReportPage_II.tex} \newpage \input{ReportPage_III.tex} \end{document} \ No newline at end of file diff --git a/examples/solver_comparison/ReportPage_I.tex b/examples/solver_comparison/ReportPage_I.tex index ce7a3f6..4c1e286 100644 --- a/examples/solver_comparison/ReportPage_I.tex +++ b/examples/solver_comparison/ReportPage_I.tex @@ -1,138 +1,151 @@ \subsection{Virtual Function I} The figures below are depicting the axis plots of the virtual hyperparameter space function I to be optimized with the available hyppopy solvers. \begin{figure}[h] \begin{subfigure}{0.32\textwidth} \includegraphics[width=\linewidth]{gt_I/axis_00} \caption{axis 00} \label{fig:axis00_I} \end{subfigure} \begin{subfigure}{0.32\textwidth} \includegraphics[width=\linewidth]{gt_I/axis_01} \caption{axis 01} \label{fig:axis01_I} \end{subfigure} \begin{subfigure}{0.32\textwidth} \includegraphics[width=\linewidth]{gt_I/axis_02} \caption{axis 02} \label{fig:axis02_I} \end{subfigure} \end{figure} \begin{figure}[h] \begin{subfigure}{0.32\textwidth} \includegraphics[width=\linewidth]{gt_I/axis_03} \caption{axis 03} \label{fig:axis03_I} \end{subfigure} \begin{subfigure}{0.32\textwidth} \includegraphics[width=\linewidth]{gt_I/axis_04} \caption{axis 04} \label{fig:axis04_I} \end{subfigure} \begin{subfigure}{0.32\textwidth} \includegraphics[width=\linewidth]{gt_I/dummy} \caption{} \label{fig:dummy1_I} \end{subfigure} \end{figure} \newpage \subsubsection{Minimum Finding Abilities} -The pictures below depict the accuracies reached on the individual axis. The light green region is the ground truth and the red, blue, violet and orange lines are the results after 50, 100, 250, and 500 iterations. Each line is the mean result over 50 individual optimizations on the target function. +The pictures below depict the accuracies reached on the individual axis. The light green region is the ground truth and the red, blue, violet and orange lines are the results after \myiterI, \myiterII, \myiterIII, and \myiterIV~iterations. Each line is the mean result over 50 individual optimizations on the target function. Exception is the QuasiRandomSolver, whose parameter space is deterministic and thus has no natural statistical variation. \begin{figure}[h] \begin{subfigure}{0.5\textwidth} \includegraphics[width=0.9\linewidth]{data_I/hyperopt_deviation} \label{fig:hyperopt_deviation_I} \end{subfigure} \begin{subfigure}{0.5\textwidth} \includegraphics[width=0.9\linewidth]{data_I/optunity_deviation} \label{fig:optunity_deviation_I} \end{subfigure} \end{figure} \begin{figure}[h] \begin{subfigure}{0.5\textwidth} \includegraphics[width=0.9\linewidth]{data_I/optuna_deviation} \label{fig:optuna_deviation_I} \end{subfigure} \begin{subfigure}{0.5\textwidth} \includegraphics[width=0.9\linewidth]{data_I/randomsearch_deviation} \label{fig:randomsearch_deviation_I} \end{subfigure} \end{figure} \begin{figure}[h] \begin{subfigure}{0.5\textwidth} \includegraphics[width=0.9\linewidth]{data_I/quasirandomsearch_deviation} \label{fig:quasirandomsearch_deviation_I} \end{subfigure} \begin{subfigure}{0.5\textwidth} \includegraphics[width=0.9\linewidth]{data_I/dummy} \label{fig:dummy2_I} \end{subfigure} \end{figure} \newpage \subsubsection{Relative Distances to the Axis Optima} The pictures in this section depict the mean distance and the standard deviation per axis for each solver. \begin{figure}[h] \begin{subfigure}{0.5\textwidth} - \includegraphics[width=0.9\linewidth]{data_I/errorbars_50} - \label{fig:errorbars_50_I} + \includegraphics[width=0.9\linewidth]{data_I/errorbars_\myiterI} + \label{fig:errorbars_\myiterI_I} \end{subfigure} \begin{subfigure}{0.5\textwidth} - \includegraphics[width=0.9\linewidth]{data_I/errorbars_100} - \label{fig:errorbars_100_I} + \includegraphics[width=0.9\linewidth]{data_I/errorbars_\myiterII} + \label{fig:errorbars_\myiterII_I} \end{subfigure} \end{figure} \begin{figure}[h] \begin{subfigure}{0.5\textwidth} - \includegraphics[width=0.9\linewidth]{data_I/errorbars_250} - \label{fig:errorbars_250_I} + \includegraphics[width=0.9\linewidth]{data_I/errorbars_\myiterIII} + \label{fig:errorbars_\myiterIII_I} \end{subfigure} \begin{subfigure}{0.5\textwidth} - \includegraphics[width=0.9\linewidth]{data_I/errorbars_500} - \label{fig:errorbars_500_I} + \includegraphics[width=0.9\linewidth]{data_I/errorbars_\myiterIV} + \label{fig:errorbars_\myiterIV_I} \end{subfigure} \end{figure} \newpage \subsubsection{Convergence Beahviour} The pictures in this section depict the loss over iteration plots for each of the 50 iterations for each solver. For better visualization the Loss values are sorted, so the mapping between Iteration and Loss values might not be correct. The purpose of these plots is to show the overall Loss curve for each solver and it's variation over different runs. \begin{figure}[h] \begin{subfigure}{0.5\textwidth} - \includegraphics[width=0.9\linewidth]{data_I/losshistory_50} - \label{fig:losshistory_50_I} + \includegraphics[width=0.9\linewidth]{data_I/losshistory_\myiterI} + \label{fig:losshistory_\myiterI_I} \end{subfigure} \begin{subfigure}{0.5\textwidth} - \includegraphics[width=0.9\linewidth]{data_I/losshistory_100} - \label{fig:losshistory_100_I} + \includegraphics[width=0.9\linewidth]{data_I/losshistory_\myiterII} + \label{fig:losshistory_\myiterII_I} \end{subfigure} \end{figure} \begin{figure}[h] \begin{subfigure}{0.5\textwidth} - \includegraphics[width=0.9\linewidth]{data_I/losshistory_250} - \label{fig:losshistory_250_I} + \includegraphics[width=0.9\linewidth]{data_I/losshistory_\myiterIII} + \label{fig:losshistory_\myiterIII_I} \end{subfigure} \begin{subfigure}{0.5\textwidth} - \includegraphics[width=0.9\linewidth]{data_I/losshistory_500} - \label{fig:losshistory_500_I} + \includegraphics[width=0.9\linewidth]{data_I/losshistory_\myiterIV} + \label{fig:losshistory_\myiterIV_I} \end{subfigure} -\end{figure} \ No newline at end of file +\end{figure} + + +\newpage + + +\subsubsection{Mean Solver Runtime} + +The below depicts the mean computation time per iteration for each solver. The time axis is logarithmic! + +\begin{figure}[h] + \includegraphics[width=\linewidth]{data_I/durations} + \label{fig:duration_I} +\end{figure} diff --git a/examples/solver_comparison/ReportPage_II.tex b/examples/solver_comparison/ReportPage_II.tex index 2d3198b..6c0989d 100644 --- a/examples/solver_comparison/ReportPage_II.tex +++ b/examples/solver_comparison/ReportPage_II.tex @@ -1,138 +1,151 @@ \subsection{Virtual Function II} The figures below are depicting the axis plots of the virtual hyperparameter space function II to be optimized with the available hyppopy solvers. \begin{figure}[h] \begin{subfigure}{0.32\textwidth} \includegraphics[width=\linewidth]{gt_II/axis_00} \caption{axis 00} \label{fig:axis00_II} \end{subfigure} \begin{subfigure}{0.32\textwidth} \includegraphics[width=\linewidth]{gt_II/axis_01} \caption{axis 01} \label{fig:axis01_II} \end{subfigure} \begin{subfigure}{0.32\textwidth} \includegraphics[width=\linewidth]{gt_II/axis_02} \caption{axis 02} \label{fig:axis02_II} \end{subfigure} \end{figure} \begin{figure}[h] \begin{subfigure}{0.32\textwidth} \includegraphics[width=\linewidth]{gt_II/axis_03} \caption{axis 03} \label{fig:axis03_II} \end{subfigure} \begin{subfigure}{0.32\textwidth} \includegraphics[width=\linewidth]{gt_II/axis_04} \caption{axis 04} \label{fig:axis04_II} \end{subfigure} \begin{subfigure}{0.32\textwidth} \includegraphics[width=\linewidth]{gt_II/dummy} \caption{} \label{fig:dummy1_II} \end{subfigure} \end{figure} \newpage \subsubsection{Minimum Finding Abilities} -The pictures below depict the accuracies reached on the individual axis. The light green region is the ground truth and the red, blue, violet and orange lines are the results after 50, 100, 250, and 500 iterations. Each line is the mean result over 50 individual optimizations on the target function. +The pictures below depict the accuracies reached on the individual axis. The light green region is the ground truth and the red, blue, violet and orange lines are the results after \myiterI, \myiterII, \myiterIII, and \myiterIV~iterations. Each line is the mean result over 50 individual optimizations on the target function. Exception is the QuasiRandomSolver, whose parameter space is deterministic and thus has no natural statistical variation. \begin{figure}[h] \begin{subfigure}{0.5\textwidth} \includegraphics[width=0.9\linewidth]{data_II/hyperopt_deviation} \label{fig:hyperopt_deviation_II} \end{subfigure} \begin{subfigure}{0.5\textwidth} \includegraphics[width=0.9\linewidth]{data_II/optunity_deviation} \label{fig:optunity_deviation_II} \end{subfigure} \end{figure} \begin{figure}[h] \begin{subfigure}{0.5\textwidth} \includegraphics[width=0.9\linewidth]{data_II/optuna_deviation} \label{fig:optuna_deviation_II} \end{subfigure} \begin{subfigure}{0.5\textwidth} \includegraphics[width=0.9\linewidth]{data_II/randomsearch_deviation} \label{fig:randomsearch_deviation_II} \end{subfigure} \end{figure} \begin{figure}[h] \begin{subfigure}{0.5\textwidth} \includegraphics[width=0.9\linewidth]{data_II/quasirandomsearch_deviation} \label{fig:quasirandomsearch_deviation_II} \end{subfigure} \begin{subfigure}{0.5\textwidth} \includegraphics[width=0.9\linewidth]{data_II/dummy} \label{fig:dummy2_II} \end{subfigure} \end{figure} \newpage \subsubsection{Relative Distances to the Axis Optima} The pictures in this section depict the mean distance and the standard deviation per axis for each solver. \begin{figure}[h] \begin{subfigure}{0.5\textwidth} - \includegraphics[width=0.9\linewidth]{data_II/errorbars_50} - \label{fig:errorbars_50_II} + \includegraphics[width=0.9\linewidth]{data_II/errorbars_\myiterI} + \label{fig:errorbars_\myiterI_II} \end{subfigure} \begin{subfigure}{0.5\textwidth} - \includegraphics[width=0.9\linewidth]{data_II/errorbars_100} - \label{fig:errorbars_100_II} + \includegraphics[width=0.9\linewidth]{data_II/errorbars_\myiterII} + \label{fig:errorbars_\myiterII_II} \end{subfigure} \end{figure} \begin{figure}[h] \begin{subfigure}{0.5\textwidth} - \includegraphics[width=0.9\linewidth]{data_II/errorbars_250} - \label{fig:errorbars_250_II} + \includegraphics[width=0.9\linewidth]{data_II/errorbars_\myiterIII} + \label{fig:errorbars_\myiterIII_II} \end{subfigure} \begin{subfigure}{0.5\textwidth} - \includegraphics[width=0.9\linewidth]{data_II/errorbars_500} - \label{fig:errorbars_500_II} + \includegraphics[width=0.9\linewidth]{data_II/errorbars_\myiterIV} + \label{fig:errorbars_\myiterIV_II} \end{subfigure} \end{figure} \newpage \subsubsection{Convergence Beahviour} The pictures in this section depict the loss over iteration plots for each of the 50 iterations for each solver. For better visualization the Loss values are sorted, so the mapping between Iteration and Loss values might not be correct. The purpose of these plots is to show the overall Loss curve for each solver and it's variation over different runs. \begin{figure}[h] \begin{subfigure}{0.5\textwidth} - \includegraphics[width=0.9\linewidth]{data_II/losshistory_50} - \label{fig:losshistory_50_II} + \includegraphics[width=0.9\linewidth]{data_II/losshistory_\myiterI} + \label{fig:losshistory_\myiterI_II} \end{subfigure} \begin{subfigure}{0.5\textwidth} - \includegraphics[width=0.9\linewidth]{data_II/losshistory_100} - \label{fig:losshistory_100_II} + \includegraphics[width=0.9\linewidth]{data_II/losshistory_\myiterII} + \label{fig:losshistory_\myiterII_II} \end{subfigure} \end{figure} \begin{figure}[h] \begin{subfigure}{0.5\textwidth} - \includegraphics[width=0.9\linewidth]{data_II/losshistory_250} - \label{fig:losshistory_250_II} + \includegraphics[width=0.9\linewidth]{data_II/losshistory_\myiterIII} + \label{fig:losshistory_\myiterIII_II} \end{subfigure} \begin{subfigure}{0.5\textwidth} - \includegraphics[width=0.9\linewidth]{data_II/losshistory_500} - \label{fig:losshistory_500_II} + \includegraphics[width=0.9\linewidth]{data_II/losshistory_\myiterIV} + \label{fig:losshistory_\myiterIV_II} \end{subfigure} +\end{figure} + + +\newpage + + +\subsubsection{Mean Solver Runtime} + +The below depicts the mean computation time per iteration for each solver. The time axis is logarithmic! + +\begin{figure}[h] + \includegraphics[width=\linewidth]{data_II/durations} + \label{fig:duration_II} \end{figure} \ No newline at end of file diff --git a/examples/solver_comparison/ReportPage_III.tex b/examples/solver_comparison/ReportPage_III.tex index 06b1e29..a6b34e3 100644 --- a/examples/solver_comparison/ReportPage_III.tex +++ b/examples/solver_comparison/ReportPage_III.tex @@ -1,138 +1,151 @@ \subsection{Virtual Function III} The figures below are depicting the axis plots of the virtual hyperparameter space function III to be optimized with the available hyppopy solvers. \begin{figure}[h] \begin{subfigure}{0.32\textwidth} \includegraphics[width=\linewidth]{gt_III/axis_00} \caption{axis 00} \label{fig:axis00_III} \end{subfigure} \begin{subfigure}{0.32\textwidth} \includegraphics[width=\linewidth]{gt_III/axis_01} \caption{axis 01} \label{fig:axis01_III} \end{subfigure} \begin{subfigure}{0.32\textwidth} \includegraphics[width=\linewidth]{gt_III/axis_02} \caption{axis 02} \label{fig:axis02_III} \end{subfigure} \end{figure} \begin{figure}[h] \begin{subfigure}{0.32\textwidth} \includegraphics[width=\linewidth]{gt_III/axis_03} \caption{axis 03} \label{fig:axis03_III} \end{subfigure} \begin{subfigure}{0.32\textwidth} \includegraphics[width=\linewidth]{gt_III/axis_04} \caption{axis 04} \label{fig:axis04_III} \end{subfigure} \begin{subfigure}{0.32\textwidth} \includegraphics[width=\linewidth]{gt_III/dummy} \caption{} \label{fig:dummy1_III} \end{subfigure} \end{figure} \newpage \subsubsection{Minimum Finding Abilities} -The pictures below depict the accuracies reached on the individual axis. The light green region is the ground truth and the red, blue, violet and orange lines are the results after 50, 100, 250, and 500 iterations. Each line is the mean result over 50 individual optimizations on the target function. +The pictures below depict the accuracies reached on the individual axis. The light green region is the ground truth and the red, blue, violet and orange lines are the results after \myiterI, \myiterII, \myiterIII, and \myiterIV~iterations. Each line is the mean result over 50 individual optimizations on the target function. Exception is the QuasiRandomSolver, whose parameter space is deterministic and thus has no natural statistical variation. \begin{figure}[h] \begin{subfigure}{0.5\textwidth} \includegraphics[width=0.9\linewidth]{data_III/hyperopt_deviation} \label{fig:hyperopt_deviation_III} \end{subfigure} \begin{subfigure}{0.5\textwidth} \includegraphics[width=0.9\linewidth]{data_III/optunity_deviation} \label{fig:optunity_deviation_III} \end{subfigure} \end{figure} \begin{figure}[h] \begin{subfigure}{0.5\textwidth} \includegraphics[width=0.9\linewidth]{data_III/optuna_deviation} \label{fig:optuna_deviation_III} \end{subfigure} \begin{subfigure}{0.5\textwidth} \includegraphics[width=0.9\linewidth]{data_III/randomsearch_deviation} \label{fig:randomsearch_deviation_III} \end{subfigure} \end{figure} \begin{figure}[h] \begin{subfigure}{0.5\textwidth} \includegraphics[width=0.9\linewidth]{data_III/quasirandomsearch_deviation} \label{fig:quasirandomsearch_deviation_III} \end{subfigure} \begin{subfigure}{0.5\textwidth} \includegraphics[width=0.9\linewidth]{data_III/dummy} \label{fig:dummy2_III} \end{subfigure} \end{figure} \newpage \subsubsection{Relative Distances to the Axis Optima} The pictures in this section depict the mean distance and the standard deviation per axis for each solver. \begin{figure}[h] \begin{subfigure}{0.5\textwidth} - \includegraphics[width=0.9\linewidth]{data_III/errorbars_50} - \label{fig:errorbars_50_III} + \includegraphics[width=0.9\linewidth]{data_III/errorbars_\myiterI} + \label{fig:errorbars_\myiterI_III} \end{subfigure} \begin{subfigure}{0.5\textwidth} - \includegraphics[width=0.9\linewidth]{data_III/errorbars_100} - \label{fig:errorbars_100_III} + \includegraphics[width=0.9\linewidth]{data_III/errorbars_\myiterII} + \label{fig:errorbars_\myiterII_III} \end{subfigure} \end{figure} \begin{figure}[h] \begin{subfigure}{0.5\textwidth} - \includegraphics[width=0.9\linewidth]{data_III/errorbars_250} - \label{fig:errorbars_250_III} + \includegraphics[width=0.9\linewidth]{data_III/errorbars_\myiterIII} + \label{fig:errorbars_\myiterIII_III} \end{subfigure} \begin{subfigure}{0.5\textwidth} - \includegraphics[width=0.9\linewidth]{data_III/errorbars_500} - \label{fig:errorbars_500_III} + \includegraphics[width=0.9\linewidth]{data_III/errorbars_\myiterIV} + \label{fig:errorbars_\myiterIV_III} \end{subfigure} \end{figure} \newpage \subsubsection{Convergence Beahviour} The pictures in this section depict the loss over iteration plots for each of the 50 iterations for each solver. For better visualization the Loss values are sorted, so the mapping between Iteration and Loss values might not be correct. The purpose of these plots is to show the overall Loss curve for each solver and it's variation over different runs. \begin{figure}[h] \begin{subfigure}{0.5\textwidth} - \includegraphics[width=0.9\linewidth]{data_III/losshistory_50} - \label{fig:losshistory_50_III} + \includegraphics[width=0.9\linewidth]{data_III/losshistory_\myiterI} + \label{fig:losshistory_\myiterI_III} \end{subfigure} \begin{subfigure}{0.5\textwidth} - \includegraphics[width=0.9\linewidth]{data_III/losshistory_100} - \label{fig:losshistory_100_III} + \includegraphics[width=0.9\linewidth]{data_III/losshistory_\myiterII} + \label{fig:losshistory_\myiterII_III} \end{subfigure} \end{figure} \begin{figure}[h] \begin{subfigure}{0.5\textwidth} - \includegraphics[width=0.9\linewidth]{data_III/losshistory_250} - \label{fig:losshistory_250_III} + \includegraphics[width=0.9\linewidth]{data_III/losshistory_\myiterIII} + \label{fig:losshistory_\myiterIII_III} \end{subfigure} \begin{subfigure}{0.5\textwidth} - \includegraphics[width=0.9\linewidth]{data_III/losshistory_500} - \label{fig:losshistory_500_III} + \includegraphics[width=0.9\linewidth]{data_III/losshistory_\myiterIV} + \label{fig:losshistory_\myiterIV_III} \end{subfigure} +\end{figure} + + +\newpage + + +\subsubsection{Mean Solver Runtime} + +The below depicts the mean computation time per iteration for each solver. The time axis is logarithmic! + +\begin{figure}[h] + \includegraphics[width=\linewidth]{data_III/durations} + \label{fig:duration_III} \end{figure} \ No newline at end of file diff --git a/examples/solver_comparison/gfx/gt_I/axis_00.png b/examples/solver_comparison/gfx/gt_I/axis_00.png new file mode 100644 index 0000000..73c23aa Binary files /dev/null and b/examples/solver_comparison/gfx/gt_I/axis_00.png differ diff --git a/examples/solver_comparison/gfx/gt_I/axis_01.png b/examples/solver_comparison/gfx/gt_I/axis_01.png new file mode 100644 index 0000000..a060b58 Binary files /dev/null and b/examples/solver_comparison/gfx/gt_I/axis_01.png differ diff --git a/examples/solver_comparison/gfx/gt_I/axis_02.png b/examples/solver_comparison/gfx/gt_I/axis_02.png new file mode 100644 index 0000000..2795d90 Binary files /dev/null and b/examples/solver_comparison/gfx/gt_I/axis_02.png differ diff --git a/examples/solver_comparison/gfx/gt_I/axis_03.png b/examples/solver_comparison/gfx/gt_I/axis_03.png new file mode 100644 index 0000000..4ad23e3 Binary files /dev/null and b/examples/solver_comparison/gfx/gt_I/axis_03.png differ diff --git a/examples/solver_comparison/gfx/gt_I/axis_04.png b/examples/solver_comparison/gfx/gt_I/axis_04.png new file mode 100644 index 0000000..38facdb Binary files /dev/null and b/examples/solver_comparison/gfx/gt_I/axis_04.png differ diff --git a/examples/solver_comparison/gfx/gt_I/dummy.png b/examples/solver_comparison/gfx/gt_I/dummy.png new file mode 100644 index 0000000..67f21c9 Binary files /dev/null and b/examples/solver_comparison/gfx/gt_I/dummy.png differ diff --git a/examples/solver_comparison/gfx/gt_II/axis_00.png b/examples/solver_comparison/gfx/gt_II/axis_00.png new file mode 100644 index 0000000..5f15a11 Binary files /dev/null and b/examples/solver_comparison/gfx/gt_II/axis_00.png differ diff --git a/examples/solver_comparison/gfx/gt_II/axis_01.png b/examples/solver_comparison/gfx/gt_II/axis_01.png new file mode 100644 index 0000000..15df27d Binary files /dev/null and b/examples/solver_comparison/gfx/gt_II/axis_01.png differ diff --git a/examples/solver_comparison/gfx/gt_II/axis_02.png b/examples/solver_comparison/gfx/gt_II/axis_02.png new file mode 100644 index 0000000..2277653 Binary files /dev/null and b/examples/solver_comparison/gfx/gt_II/axis_02.png differ diff --git a/examples/solver_comparison/gfx/gt_II/axis_03.png b/examples/solver_comparison/gfx/gt_II/axis_03.png new file mode 100644 index 0000000..a082dea Binary files /dev/null and b/examples/solver_comparison/gfx/gt_II/axis_03.png differ diff --git a/examples/solver_comparison/gfx/gt_II/axis_04.png b/examples/solver_comparison/gfx/gt_II/axis_04.png new file mode 100644 index 0000000..6bf90f7 Binary files /dev/null and b/examples/solver_comparison/gfx/gt_II/axis_04.png differ diff --git a/examples/solver_comparison/gfx/gt_II/dummy.png b/examples/solver_comparison/gfx/gt_II/dummy.png new file mode 100644 index 0000000..67f21c9 Binary files /dev/null and b/examples/solver_comparison/gfx/gt_II/dummy.png differ diff --git a/examples/solver_comparison/gfx/gt_III/axis_00.png b/examples/solver_comparison/gfx/gt_III/axis_00.png new file mode 100644 index 0000000..009c5e6 Binary files /dev/null and b/examples/solver_comparison/gfx/gt_III/axis_00.png differ diff --git a/examples/solver_comparison/gfx/gt_III/axis_01.png b/examples/solver_comparison/gfx/gt_III/axis_01.png new file mode 100644 index 0000000..10ee1b2 Binary files /dev/null and b/examples/solver_comparison/gfx/gt_III/axis_01.png differ diff --git a/examples/solver_comparison/gfx/gt_III/axis_02.png b/examples/solver_comparison/gfx/gt_III/axis_02.png new file mode 100644 index 0000000..7a099f2 Binary files /dev/null and b/examples/solver_comparison/gfx/gt_III/axis_02.png differ diff --git a/examples/solver_comparison/gfx/gt_III/axis_03.png b/examples/solver_comparison/gfx/gt_III/axis_03.png new file mode 100644 index 0000000..1667f1c Binary files /dev/null and b/examples/solver_comparison/gfx/gt_III/axis_03.png differ diff --git a/examples/solver_comparison/gfx/gt_III/axis_04.png b/examples/solver_comparison/gfx/gt_III/axis_04.png new file mode 100644 index 0000000..3e6dec7 Binary files /dev/null and b/examples/solver_comparison/gfx/gt_III/axis_04.png differ diff --git a/examples/solver_comparison/gfx/gt_III/dummy.png b/examples/solver_comparison/gfx/gt_III/dummy.png new file mode 100644 index 0000000..67f21c9 Binary files /dev/null and b/examples/solver_comparison/gfx/gt_III/dummy.png differ diff --git a/examples/tutorial_multisolver.py b/examples/tutorial_multisolver.py index 5f9acc4..c1e0d96 100644 --- a/examples/tutorial_multisolver.py +++ b/examples/tutorial_multisolver.py @@ -1,183 +1,183 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE # In this tutorial we solve an optimization problem using the Hyperopt Solver (http://hyperopt.github.io/hyperopt/). # Hyperopt uses a Baysian - Tree Parzen Estimator - Optimization approach, which means that each iteration computes a # new function value of the blackbox, interpolates a guess for the whole energy function and predicts a point to # compute the next function value at. This next point is not necessarily a "better" value, it's only the value with # the highest uncertainty for the function interpolation. # # See a visual explanation e.g. here (http://philipperemy.github.io/visualization/) # import the HyppopyProject class keeping track of inputs from hyppopy.HyppopyProject import HyppopyProject # import the SolverPool singleton class from hyppopy.SolverPool import SolverPool # import the Blackboxfunction class wrapping your problem for Hyppopy from hyppopy.BlackboxFunction import BlackboxFunction # Next step is defining the problem space and all settings Hyppopy needs to optimize your problem. # The config is a simple nested dictionary with two obligatory main sections, hyperparameter and settings. # The hyperparameter section defines your searchspace. Each hyperparameter is again a dictionary with: # # - a domain ['categorical', 'uniform', 'normal', 'loguniform'] # - the domain data [left bound, right bound] and # - a type of your domain ['str', 'int', 'float'] # # The settings section has two subcategories, solver and custom. The first contains settings for the solver, # here 'max_iterations' - is the maximum number of iteration. # # The custom section allows defining custom parameter. An entry here is transformed to a member variable of the # HyppopyProject class. These can be useful when implementing new solver classes or for control your hyppopy script. # Here we use it as a solver switch to control the usage of our solver via the config. This means with the script # below your can try out every solver by changing use_solver to 'optunity', 'randomsearch', 'gridsearch',... # It can be used like so: project.custom_use_plugin (see below) If using the gridsearch solver, max_iterations is # ignored, instead each hyperparameter must specifiy a number of samples additionally to the range like so: # 'data': [0, 1, 100] which means sampling the space from 0 to 1 in 100 intervals. config = { "hyperparameter": { "C": { "domain": "uniform", "data": [0.0001, 20], "type": float }, "gamma": { "domain": "uniform", "data": [0.0001, 20.0], "type": float }, "kernel": { "domain": "categorical", "data": ["linear", "sigmoid", "poly", "rbf"], "type": str }, "decision_function_shape": { "domain": "categorical", "data": ["ovo", "ovr"], "type": str } }, -"max_iterations": 100, +"max_iterations": 300, "solver": "quasirandomsearch" } # When creating a HyppopyProject instance we # pass the config dictionary to the constructor. project = HyppopyProject(config=config) # demonstration of the custom parameter access print("-"*30) print("max_iterations:\t{}".format(project.max_iterations)) print("solver chosen -> {}".format(project.solver)) print("-"*30) # The BlackboxFunction signature is as follows: # BlackboxFunction(blackbox_func=None, # dataloader_func=None, # preprocess_func=None, # callback_func=None, # data=None, # **kwargs) # # - blackbox_func: a function pointer to the users loss function # - dataloader_func: a function pointer for handling dataloading. The function is called once before # optimizing. What it returns is passed as first argument to your loss functions # data argument. # - preprocess_func: a function pointer for data preprocessing. The function is called once before # optimizing and gets via kwargs['data'] the raw data object set directly or returned # from dataloader_func. What this function returns is then what is passed as first # argument to your loss function. # - callback_func: a function pointer called after each iteration. The input kwargs is a dictionary # keeping the parameters used in this iteration, the 'iteration' index, the 'loss' # and the 'status'. The function in this example is used for realtime printing it's # input but can also be used for realtime visualization. # - data: if not done via dataloader_func one can set a raw_data object directly # - kwargs: dict that whose content is passed to all functions above. from sklearn.svm import SVC from sklearn.datasets import load_iris from sklearn.model_selection import cross_val_score def my_dataloader_function(**kwargs): print("Dataloading...") # kwargs['params'] allows accessing additional parameter passed, see below my_preproc_param, my_dataloader_input. print("my loading argument: {}".format(kwargs['params']['my_dataloader_input'])) iris_data = load_iris() return [iris_data.data, iris_data.target] def my_preprocess_function(**kwargs): print("Preprocessing...") # kwargs['data'] allows accessing the input data print("data:", kwargs['data'][0].shape, kwargs['data'][1].shape) # kwargs['params'] allows accessing additional parameter passed, see below my_preproc_param, my_dataloader_input. print("kwargs['params']['my_preproc_param']={}".format(kwargs['params']['my_preproc_param']), "\n") # if the preprocessing function returns something, # the input data will be replaced with the data returned by this function. x = kwargs['data'][0] y = kwargs['data'][1] for i in range(x.shape[0]): x[i, :] += kwargs['params']['my_preproc_param'] return [x, y] def my_callback_function(**kwargs): print("\r{}".format(kwargs), end="") def my_loss_function(data, params): clf = SVC(**params) return -cross_val_score(estimator=clf, X=data[0], y=data[1], cv=3).mean() # We now create the BlackboxFunction object and pass all function pointers defined above, # as well as 2 dummy parameter (my_preproc_param, my_dataloader_input) for demonstration purposes. blackbox = BlackboxFunction(blackbox_func=my_loss_function, dataloader_func=my_dataloader_function, preprocess_func=my_preprocess_function, callback_func=my_callback_function, my_preproc_param=1, my_dataloader_input='could/be/a/path') # Last step, is we use our SolverPool which automatically returns the correct solver. # There are multiple ways to get the desired solver from the solver pool. # 1. solver = SolverPool.get('hyperopt') # solver.project = project # 2. solver = SolverPool.get('hyperopt', project) # 3. The SolverPool will look for the field 'use_solver' in the project instance, if # it is present it will be used to specify the solver so that in this case it is enough # to pass the project instance. solver = SolverPool.get(project=project) # Give the solver your blackbox and run it. After execution we can get the result # via get_result() which returns a pandas dataframe containing the complete history # The dict best contains the best parameter set. solver.blackbox = blackbox #solver.start_viewer() solver.run() df, best = solver.get_results() print("\n") print("*"*100) print("Best Parameter Set:\n{}".format(best)) print("*"*100) diff --git a/hyppopy/solvers/QuasiRandomsearchSolver.py b/hyppopy/solvers/QuasiRandomsearchSolver.py index c13b5ea..28e028f 100644 --- a/hyppopy/solvers/QuasiRandomsearchSolver.py +++ b/hyppopy/solvers/QuasiRandomsearchSolver.py @@ -1,218 +1,182 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE import os import logging import warnings -import itertools import numpy as np -from random import choice from pprint import pformat from hyppopy.globals import DEBUGLEVEL from hyppopy.solvers.HyppopySolver import HyppopySolver LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) -def get_gaussian_ranges(a, b, N): - r = abs(b-a)/2 - if N % 2 == 0: - _N = int(N/2) - else: - _N = int((N-1)/2) - dr = r/_N - sigma = r/2.5 - mu = a + r - cuts = [] - csum = 0 - for n in range(_N): - x = a+r+n*dr - c = sigma*np.sqrt(2.0*np.pi)/(np.exp(-0.5*((x-mu)/sigma)**2)) - cuts.append(c) - cuts.insert(0, c) - csum += 2*c - for n in range(len(cuts)): - cuts[n] /= csum - cuts[n] *= abs(b-a) - ranges = [] - end = a - for n, c in enumerate(cuts): - start = end - end = start + c - ranges.append([start, end]) - return ranges - - def get_loguniform_ranges(a, b, N): aL = np.log(a) bL = np.log(b) exp_range = np.linspace(aL, bL, N+1) ranges = [] for i in range(N): ranges.append([np.exp(exp_range[i]), np.exp(exp_range[i+1])]) return ranges +class HaltonSequenceGenerator(object): + + def __init__(self, N_samples, dimensions): + self._N = N_samples + self._dims = dimensions + + def next_prime(self): + def is_prime(num): + "Checks if num is a prime value" + for i in range(2, int(num ** 0.5) + 1): + if (num % i) == 0: return False + return True + + prime = 3 + while 1: + if is_prime(prime): + yield prime + prime += 2 + + def vdc(self, n, base): + vdc, denom = 0, 1 + while n: + denom *= base + n, remainder = divmod(n, base) + vdc += remainder / float(denom) + return vdc + + def get_sequence(self): + seq = [] + primeGen = self.next_prime() + next(primeGen) + for d in range(self._dims): + base = next(primeGen) + seq.append([self.vdc(i, base) for i in range(self._N)]) + return seq + + class QuasiRandomSampleGenerator(object): - def __init__(self, N_samples=None, border_frac=0.1): - self._grid = None + def __init__(self, N_samples=None): self._axis = None + self._samples = [] self._numerical = [] self._categorical = [] self._N_samples = N_samples - self._border_frac = border_frac def set_axis(self, name, data, domain, dtype): if domain == "categorical": if dtype is int: data = [int(i) for i in data] elif dtype is str: data = [str(i) for i in data] elif dtype is float: data = [float(i) for i in data] self._categorical.append({"name": name, "data": data, "type": dtype}) else: self._numerical.append({"name": name, "data": data, "type": dtype, "domain": domain}) - def build_grid(self, N_samples=None): + def generate_samples(self, N_samples=None): self._axis = [] if N_samples is None: assert isinstance(self._N_samples, int), "Precondition violation, no number of samples specified!" else: self._N_samples = N_samples + axis_samples = {} if len(self._numerical) > 0: - axis_steps = int(round(self._N_samples**(1.0/len(self._numerical)))) - self._N_samples = int(axis_steps**(len(self._numerical))) - - for axis in self._numerical: - self._axis.append(None) - n = len(self._axis)-1 - boxes = None - if axis["domain"] == "uniform": - boxes = self.add_uniform_axis(n, axis_steps) - elif axis["domain"] == "normal": - boxes = self.add_normal_axis(n, axis_steps) - elif axis["domain"] == "loguniform": - boxes = self.add_loguniform_axis(n, axis_steps) - - assert isinstance(boxes, list), "failed to compute axis ranges!" - for k in range(len(boxes)): - dx = abs(boxes[k][1] - boxes[k][0]) - boxes[k][0] += self._border_frac * dx - boxes[k][1] -= self._border_frac * dx - self._axis[n] = boxes - self._grid = list(itertools.product(*self._axis)) + generator = HaltonSequenceGenerator(self._N_samples, len(self._numerical)) + unit_space = generator.get_sequence() + for n, axis in enumerate(self._numerical): + width = abs(axis["data"][1] - axis["data"][0]) + unit_space[n] = [x * width for x in unit_space[n]] + unit_space[n] = [x + axis["data"][0] for x in unit_space[n]] + if axis["type"] is int: + unit_space[n] = [int(round(x)) for x in unit_space[n]] + axis_samples[axis["name"]] = unit_space[n] else: warnings.warn("No numerical axis defined, this warning can be ignored if searchspace is categorical only, otherwise check if axis was set!") - def add_uniform_axis(self, n, axis_steps): - drange = self._numerical[n]["data"] - width = abs(drange[1]-drange[0]) - dx = width / axis_steps - boxes = [] - for k in range(1, axis_steps+1): - bl = drange[0] + (k-1)*dx - br = drange[0] + k*dx - boxes.append([bl, br]) - return boxes - - def add_normal_axis(self, n, axis_steps): - drange = self._numerical[n]["data"] - boxes = get_gaussian_ranges(drange[0], drange[1], axis_steps) - for k in range(len(boxes)): - dx = abs(boxes[k][1] - boxes[k][0]) - boxes[k][0] += self._border_frac * dx - boxes[k][1] -= self._border_frac * dx - return boxes - - def add_loguniform_axis(self, n, axis_steps): - drange = self._numerical[n]["data"] - boxes = get_loguniform_ranges(drange[0], drange[1], axis_steps) - for k in range(len(boxes)): - dx = abs(boxes[k][1] - boxes[k][0]) - boxes[k][0] += self._border_frac * dx - boxes[k][1] -= self._border_frac * dx - return boxes + for n in range(self._N_samples): + sample = {} + for name, data in axis_samples.items(): + sample[name] = data[n] + for cat in self._categorical: + choice = np.random.choice(len(cat["data"]), 1)[0] + sample[cat["name"]] = cat["data"][choice] + self._samples.append(sample) def next(self): - if self._grid is None: - self.build_grid() - if len(self._grid) == 0: + if len(self._samples) == 0: + self.generate_samples() + if len(self._samples) == 0: return None - next_index = np.random.randint(0, len(self._grid), 1)[0] - next_range = self._grid.pop(next_index) - pset = {} - for n, rng in enumerate(next_range): - name = self._numerical[n]["name"] - rnd = np.random.random() - param = rng[0] + rnd*abs(rng[1]-rng[0]) - if self._numerical[n]["type"] is int: - param = int(np.floor(param)) - pset[name] = param - for cat in self._categorical: - pset[cat["name"]] = choice(cat["data"]) - return pset + next_index = np.random.choice(len(self._samples), 1)[0] + sample = self._samples.pop(next_index) + return sample class QuasiRandomsearchSolver(HyppopySolver): """ The QuasiRandomsearchSolver class implements a quasi randomsearch optimization. The quasi randomsearch supports categorical, uniform, normal and loguniform sampling. The solver defines a grid which size and appearance depends on the max_iterations parameter and the domain. The at each grid box a random value is drawn. This ensures both, random parameter samples with the cosntraint that the space is evenly sampled and cluster building prevention.""" def __init__(self, project=None): HyppopySolver.__init__(self, project) self._sampler = None def define_interface(self): self.add_member("max_iterations", int) self.add_hyperparameter_signature(name="domain", dtype=str, - options=["uniform", "normal", "loguniform", "categorical"]) + options=["uniform", "categorical"]) self.add_hyperparameter_signature(name="data", dtype=list) self.add_hyperparameter_signature(name="type", dtype=type) def loss_function_call(self, params): loss = self.blackbox(**params) if loss is None: return np.nan return loss def execute_solver(self, searchspace): N = self.max_iterations self._sampler = QuasiRandomSampleGenerator(N) for name, axis in searchspace.items(): self._sampler.set_axis(name, axis["data"], axis["domain"], axis["type"]) try: for n in range(N): params = self._sampler.next() if params is None: break self.loss_function(**params) except Exception as e: msg = "internal error in randomsearch execute_solver occured. {}".format(e) LOG.error(msg) raise BrokenPipeError(msg) self.best = self._trials.argmin def convert_searchspace(self, hyperparameter): """ this function simply pipes the input parameter through, the sample drawing functions are responsible for interpreting the parameter. :param hyperparameter: [dict] hyperparameter space :return: [dict] hyperparameter space """ LOG.debug("convert input parameter\n\n\t{}\n".format(pformat(hyperparameter))) return hyperparameter diff --git a/hyppopy/tests/test_quasirandomsearchsolver.py b/hyppopy/tests/test_quasirandomsearchsolver.py index 86969e1..e571a92 100644 --- a/hyppopy/tests/test_quasirandomsearchsolver.py +++ b/hyppopy/tests/test_quasirandomsearchsolver.py @@ -1,235 +1,67 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE import unittest import matplotlib.pylab as plt from hyppopy.solvers.QuasiRandomsearchSolver import * from hyppopy.VirtualFunction import VirtualFunction from hyppopy.HyppopyProject import HyppopyProject class QuasiRandomsearchTestSuite(unittest.TestCase): def setUp(self): pass - def test_get_gaussian_ranges(self): - interval = [0, 10] - N = 10 - ranges = get_gaussian_ranges(interval[0], interval[1], N) - gt = [[0, 2.592443381276233], - [2.592443381276233, 3.673134565097225], - [3.673134565097225, 4.251586871937128], - [4.251586871937128, 4.6491509407201], - [4.6491509407201, 5.000000000000001], - [5.000000000000001, 5.350849059279902], - [5.350849059279902, 5.748413128062873], - [5.748413128062873, 6.326865434902777], - [6.326865434902777, 7.407556618723769], - [7.407556618723769, 10.000000000000002]] - for a, b in zip(ranges, gt): - self.assertAlmostEqual(a[0], b[0]) - self.assertAlmostEqual(a[1], b[1]) - - interval = [-100, 100] - N = 10 - ranges = get_gaussian_ranges(interval[0], interval[1], N) - gt = [[-100, -48.151132374475345], - [-48.151132374475345, -26.537308698055508], - [-26.537308698055508, -14.96826256125745], - [-14.96826256125745, -7.0169811855980315], - [-7.0169811855980315, -1.2434497875801753e-14], - [-1.2434497875801753e-14, 7.016981185598007], - [7.016981185598007, 14.968262561257426], - [14.968262561257426, 26.537308698055483], - [26.537308698055483, 48.151132374475324], - [48.151132374475324, 99.99999999999997]] - for a, b in zip(ranges, gt): - self.assertAlmostEqual(a[0], b[0]) - self.assertAlmostEqual(a[1], b[1]) - - def test_get_loguniform_ranges(self): - interval = [1, 1000] - N = 10 - ranges = get_loguniform_ranges(interval[0], interval[1], N) - gt = [[1.0, 1.9952623149688797], - [1.9952623149688797, 3.9810717055349727], - [3.9810717055349727, 7.943282347242818], - [7.943282347242818, 15.848931924611136], - [15.848931924611136, 31.62277660168379], - [31.62277660168379, 63.095734448019364], - [63.095734448019364, 125.89254117941677], - [125.89254117941677, 251.18864315095806], - [251.18864315095806, 501.1872336272723], - [501.1872336272723, 999.9999999999998]] - for a, b in zip(ranges, gt): - self.assertAlmostEqual(a[0], b[0]) - self.assertAlmostEqual(a[1], b[1]) - - interval = [1, 10000] - N = 50 - ranges = get_loguniform_ranges(interval[0], interval[1], N) - gt = [[1.0, 1.202264434617413], - [1.202264434617413, 1.4454397707459274], - [1.4454397707459274, 1.7378008287493756], - [1.7378008287493756, 2.0892961308540396], - [2.0892961308540396, 2.51188643150958], - [2.51188643150958, 3.0199517204020165], - [3.0199517204020165, 3.6307805477010135], - [3.6307805477010135, 4.36515832240166], - [4.36515832240166, 5.248074602497727], - [5.248074602497727, 6.309573444801933], - [6.309573444801933, 7.5857757502918375], - [7.5857757502918375, 9.120108393559098], - [9.120108393559098, 10.964781961431854], - [10.964781961431854, 13.18256738556407], - [13.18256738556407, 15.848931924611136], - [15.848931924611136, 19.054607179632477], - [19.054607179632477, 22.908676527677738], - [22.908676527677738, 27.542287033381676], - [27.542287033381676, 33.11311214825911], - [33.11311214825911, 39.810717055349734], - [39.810717055349734, 47.863009232263856], - [47.863009232263856, 57.543993733715695], - [57.543993733715695, 69.18309709189366], - [69.18309709189366, 83.17637711026713], - [83.17637711026713, 100.00000000000004], - [100.00000000000004, 120.22644346174135], - [120.22644346174135, 144.54397707459285], - [144.54397707459285, 173.78008287493753], - [173.78008287493753, 208.92961308540396], - [208.92961308540396, 251.18864315095806], - [251.18864315095806, 301.9951720402017], - [301.9951720402017, 363.0780547701015], - [363.0780547701015, 436.5158322401662], - [436.5158322401662, 524.8074602497729], - [524.8074602497729, 630.9573444801938], - [630.9573444801938, 758.5775750291845], - [758.5775750291845, 912.0108393559099], - [912.0108393559099, 1096.4781961431854], - [1096.4781961431854, 1318.2567385564075], - [1318.2567385564075, 1584.8931924611143], - [1584.8931924611143, 1905.4607179632485], - [1905.4607179632485, 2290.867652767775], - [2290.867652767775, 2754.228703338169], - [2754.228703338169, 3311.3112148259115], - [3311.3112148259115, 3981.071705534977], - [3981.071705534977, 4786.300923226385], - [4786.300923226385, 5754.399373371577], - [5754.399373371577, 6918.309709189369], - [6918.309709189369, 8317.63771102671], - [8317.63771102671, 10000.00000000001]] - for a, b in zip(ranges, gt): - self.assertAlmostEqual(a[0], b[0]) - self.assertAlmostEqual(a[1], b[1]) - - def test_QuasiRandomSampleGenerator(self): - N_samples = 10*10*10 - axis_data = {"p1": {"domain": "loguniform", "data": [1, 10000], "type": float}, - "p2": {"domain": "normal", "data": [-5, 5], "type": float}, - "p3": {"domain": "uniform", "data": [0, 10], "type": float}, - "p4": {"domain": "categorical", "data": [False, True], "type": bool}} - sampler = QuasiRandomSampleGenerator(N_samples, 0.1) - for name, axis in axis_data.items(): - sampler.set_axis(name, axis["data"], axis["domain"], axis["type"]) - - for i in range(N_samples): - sample = sampler.next() - self.assertTrue(len(sample.keys()) == 4) - for k in range(4): - self.assertTrue("p{}".format(k+1) in sample.keys()) - self.assertTrue(1 <= sample["p1"] <= 10000) - self.assertTrue(-5 <= sample["p2"] <= 5) - self.assertTrue(0 <= sample["p3"] <= 10) - self.assertTrue(isinstance(sample["p4"], bool)) - self.assertTrue(sampler.next() is None) - def test_solver_uniform(self): config = { "hyperparameter": { "axis_00": { "domain": "uniform", "data": [0, 800], "type": float }, "axis_01": { "domain": "uniform", "data": [-1, 1], "type": float }, "axis_02": { "domain": "uniform", "data": [0, 10], "type": float } }, "max_iterations": 300 } project = HyppopyProject(config) solver = QuasiRandomsearchSolver(project) vfunc = VirtualFunction() vfunc.load_default() solver.blackbox = vfunc solver.run(print_stats=False) df, best = solver.get_results() self.assertTrue(0 <= best['axis_00'] <= 800) self.assertTrue(-1 <= best['axis_01'] <= 1) self.assertTrue(0 <= best['axis_02'] <= 10) for status in df['status']: self.assertTrue(status) for loss in df['losses']: self.assertTrue(isinstance(loss, float)) - def test_solver_normal(self): - config = { - "hyperparameter": { - "axis_00": { - "domain": "normal", - "data": [500, 650], - "type": float - }, - "axis_01": { - "domain": "normal", - "data": [0, 1], - "type": float - }, - "axis_02": { - "domain": "normal", - "data": [4, 5], - "type": float - } - }, - "max_iterations": 500, - } - - solver = QuasiRandomsearchSolver(config) - vfunc = VirtualFunction() - vfunc.load_default() - solver.blackbox = vfunc - solver.run(print_stats=False) - df, best = solver.get_results() - self.assertTrue(500 <= best['axis_00'] <= 650) - self.assertTrue(0 <= best['axis_01'] <= 1) - self.assertTrue(4 <= best['axis_02'] <= 5) - - for status in df['status']: - self.assertTrue(status) - for loss in df['losses']: - self.assertTrue(isinstance(loss, float)) - if __name__ == '__main__': unittest.main()