diff --git a/LICENSE b/LICENSE index 221b5b7..366acc1 100644 --- a/LICENSE +++ b/LICENSE @@ -1,38 +1,38 @@ ======================================================================= Copyright (c) 2003-2012 German Cancer Research Center, -Division of Medical and Biological Informatics +Division of Medical Image Computing All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the German Cancer Research Center, nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ======================================================================= diff --git a/__main__.py b/__main__.py index 8b13789..9988b6a 100644 --- a/__main__.py +++ b/__main__.py @@ -1 +1,14 @@ - +# DKFZ +# +# +# Copyright (c) German Cancer Research Center, +# Division of Medical Image Computing. +# All rights reserved. +# +# This software is distributed WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. +# +# See LICENSE +# +# Author: Sven Wanner (s.wanner@dkfz.de) \ No newline at end of file diff --git a/examples/solver_comparison.py b/examples/solver_comparison.py index 0fe92f6..9dd2322 100644 --- a/examples/solver_comparison.py +++ b/examples/solver_comparison.py @@ -1,181 +1,196 @@ +# DKFZ +# +# +# Copyright (c) German Cancer Research Center, +# Division of Medical Image Computing. +# All rights reserved. +# +# This software is distributed WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. +# +# See LICENSE +# +# Author: Sven Wanner (s.wanner@dkfz.de) + import os import pickle import numpy as np from math import pi from pprint import pprint import matplotlib.pyplot as plt from hyppopy.SolverPool import SolverPool from hyppopy.HyppopyProject import HyppopyProject from hyppopy.VirtualFunction import VirtualFunction from hyppopy.BlackboxFunction import BlackboxFunction OUTPUTDIR = "C:\\Users\\s635r\\Desktop\\solver_comparison" SOLVER = ["hyperopt", "optunity", "randomsearch", "optuna"]#, "bayesopt"] ITERATIONS = [25, 100, 250, 500] STATREPEATS = 10 VFUNC = "5D3" OVERWRITE = False OUTPUTDIR = os.path.join(OUTPUTDIR, VFUNC) if not os.path.isdir(OUTPUTDIR): os.makedirs(OUTPUTDIR) def compute_deviation(solver_name, vfunc_id, iterations, N, fname): project = HyppopyProject() project.add_hyperparameter(name="axis_00", domain="uniform", data=[0, 1], dtype="float") project.add_hyperparameter(name="axis_01", domain="uniform", data=[0, 1], dtype="float") project.add_hyperparameter(name="axis_02", domain="uniform", data=[0, 1], dtype="float") project.add_hyperparameter(name="axis_03", domain="uniform", data=[0, 1], dtype="float") project.add_hyperparameter(name="axis_04", domain="uniform", data=[0, 1], dtype="float") vfunc = VirtualFunction() vfunc.load_default(vfunc_id) minima = vfunc.minima() def my_loss_function(data, params): return vfunc(**params) blackbox = BlackboxFunction(data=[], blackbox_func=my_loss_function) results = {} results["gt"] = [] for mini in minima: results["gt"].append(np.median(mini[0])) for iter in iterations: results[iter] = {"minima": {}, "loss": None} for i in range(vfunc.dims()): results[iter]["minima"]["axis_0{}".format(i)] = [] project.add_settings(section="solver", name="max_iterations", value=iter) project.add_settings(section="custom", name="use_solver", value=solver_name) solver = SolverPool.get(project=project) solver.blackbox = blackbox axis_minima = [] best_losses = [] for i in range(vfunc.dims()): axis_minima.append([]) for n in range(N): print("\rSolver={} iteration={} round={}".format(solver, iter, n), end="") solver.run(print_stats=False) df, best = solver.get_results() best_row = df['losses'].idxmin() best_losses.append(df['losses'][best_row]) for i in range(vfunc.dims()): tmp = df['axis_0{}'.format(i)][best_row] axis_minima[i].append(tmp) for i in range(vfunc.dims()): results[iter]["minima"]["axis_0{}".format(i)] = [np.mean(axis_minima[i]), np.std(axis_minima[i])] results[iter]["loss"] = [np.mean(best_losses), np.std(best_losses)] file = open(fname, 'wb') pickle.dump(results, file) file.close() def make_radarplot(results, title, fname=None): gt = results.pop("gt") categories = list(results[list(results.keys())[0]]["minima"].keys()) N = len(categories) angles = [n / float(N) * 2 * pi for n in range(N)] angles += angles[:1] ax = plt.subplot(1, 1, 1, polar=True, ) ax.set_theta_offset(pi / 2) ax.set_theta_direction(-1) plt.xticks(angles[:-1], categories, color='grey', size=8) ax.set_rlabel_position(0) plt.yticks([0.2, 0.4, 0.6, 0.8, 1.0], ["0.2", "0.4", "0.6", "0.8", "1.0"], color="grey", size=7) plt.ylim(0, 1) gt += gt[:1] ax.fill(angles, gt, color=(0.2, 0.8, 0.2), alpha=0.2) colors = [] cm = plt.get_cmap('Set1') if len(results) > 2: indices = list(range(0, len(results) + 1)) indices.pop(2) else: indices = list(range(0, len(results))) for i in range(len(results)): colors.append(cm(indices[i])) for iter, data in results.items(): values = [] for i in range(len(categories)): values.append(data["minima"]["axis_0{}".format(i)][0]) values += values[:1] color = colors.pop(0) ax.plot(angles, values, color=color, linewidth=2, linestyle='solid', label="iterations {}".format(iter)) plt.title(title, size=11, color=(0.1, 0.1, 0.1), y=1.1) plt.legend(bbox_to_anchor=(0.08, 1.12)) if fname is None: plt.show() else: plt.savefig(fname + ".png") plt.savefig(fname + ".svg") plt.clf() def make_deviationerrorplot(fnames): results = {} for fname in fnames: file = open(fname, 'rb') result = pickle.load(file) file.close() results[os.path.basename(fname)] = result pprint(results) plt.figure() for iter in results["hyperopt"].keys(): y = [] if iter == "gt": x = list(range(len(results["hyperopt"][iter]))) for i in range(len(results["hyperopt"][iter])): y.append(results["hyperopt"][iter][i]) plt.plot(x, y, "--g", label="groundtruth: {}".format(iter)) continue x = list(range(len(results["hyperopt"][iter]["minima"]))) for i in range(len(results["hyperopt"][iter]["minima"])): y.append(results["hyperopt"][iter]["minima"]["axis_0{}".format(i)][0]) plt.plot(x, y, label="iterations: {}".format(iter)) plt.title("") plt.legend() plt.show() ################################################## ############### create datasets ################## fnames = [] for solver_name in SOLVER: fname = os.path.join(OUTPUTDIR, solver_name) fnames.append(fname) if OVERWRITE or not os.path.isfile(fname): compute_deviation(solver_name, VFUNC, ITERATIONS, N=STATREPEATS, fname=fname) ################################################## ################################################## ################################################## ############## create radarplots ################# for solver_name, fname in zip(SOLVER, fnames): file = open(fname, 'rb') results = pickle.load(file) file.close() make_radarplot(results, solver_name, fname + "_deviation") ################################################## ################################################## diff --git a/examples/tutorial_custom_visualization.py b/examples/tutorial_custom_visualization.py index 9237f13..cf463c1 100644 --- a/examples/tutorial_custom_visualization.py +++ b/examples/tutorial_custom_visualization.py @@ -1,92 +1,107 @@ +# DKFZ +# +# +# Copyright (c) German Cancer Research Center, +# Division of Medical Image Computing. +# All rights reserved. +# +# This software is distributed WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. +# +# See LICENSE +# +# Author: Sven Wanner (s.wanner@dkfz.de) + import matplotlib.pylab as plt from hyppopy.SolverPool import SolverPool from hyppopy.HyppopyProject import HyppopyProject from hyppopy.VirtualFunction import VirtualFunction from hyppopy.BlackboxFunction import BlackboxFunction project = HyppopyProject() project.add_hyperparameter(name="axis_00", domain="uniform", data=[0, 1], dtype="float") project.add_hyperparameter(name="axis_01", domain="uniform", data=[0, 800], dtype="float") project.add_hyperparameter(name="axis_02", domain="uniform", data=[0, 5], dtype="float") project.add_hyperparameter(name="axis_03", domain="uniform", data=[1, 10000], dtype="float") project.add_hyperparameter(name="axis_04", domain="uniform", data=[0, 10], dtype="float") project.add_settings(section="solver", name="max_iterations", value=500) project.add_settings(section="custom", name="use_solver", value="randomsearch") plt.ion() fig, axes = plt.subplots(nrows=2, ncols=3, figsize=(12, 8), sharey=True) plot_data = {"iterations": [], "loss": [], "axis_00": [], "axis_01": [], "axis_02": [], "axis_03": [], "axis_04": []} def my_visualization_function(**kwargs): print("\r{}".format(kwargs), end="") plot_data["iterations"].append(kwargs['iterations']) plot_data["loss"].append(kwargs['loss']) plot_data["axis_00"].append(kwargs['axis_00']) plot_data["axis_01"].append(kwargs['axis_01']) plot_data["axis_02"].append(kwargs['axis_02']) plot_data["axis_03"].append(kwargs['axis_03']) plot_data["axis_04"].append(kwargs['axis_04']) axes[0, 0].clear() axes[0, 0].scatter(plot_data["axis_00"], plot_data["loss"], c=plot_data["loss"], cmap="jet", marker='.') axes[0, 0].set_ylabel("loss") axes[0, 0].set_xlabel("axis_00") axes[0, 1].clear() axes[0, 1].scatter(plot_data["axis_01"], plot_data["loss"], c=plot_data["loss"], cmap="jet", marker='.') axes[0, 1].set_xlabel("axis_01") axes[0, 2].clear() axes[0, 2].scatter(plot_data["axis_02"], plot_data["loss"], c=plot_data["loss"], cmap="jet", marker='.') axes[0, 2].set_xlabel("axis_02") axes[1, 0].clear() axes[1, 0].scatter(plot_data["axis_03"], plot_data["loss"], c=plot_data["loss"], cmap="jet", marker='.') axes[1, 0].set_ylabel("loss") axes[1, 0].set_xlabel("axis_03") axes[1, 1].clear() axes[1, 1].scatter(plot_data["axis_04"], plot_data["loss"], c=plot_data["loss"], cmap="jet", marker='.') axes[1, 1].set_xlabel("axis_04") axes[1, 2].clear() axes[1, 2].plot(plot_data["iterations"], plot_data["loss"], "--", c=(0.8, 0.8, 0.8, 0.5)) axes[1, 2].scatter(plot_data["iterations"], plot_data["loss"], marker='.', c=(0.2, 0.2, 0.2)) axes[1, 2].set_xlabel("iterations") plt.draw() plt.tight_layout() plt.pause(0.001) def my_loss_function(data, params): vfunc = VirtualFunction() vfunc.load_default(5) return vfunc(**params) blackbox = BlackboxFunction(data=[], blackbox_func=my_loss_function, callback_func=my_visualization_function) solver = SolverPool.get(project=project) solver.blackbox = blackbox solver.run() df, best = solver.get_results() print("\n") print("*" * 100) print("Best Parameter Set:\n{}".format(best)) print("*" * 100) print("") save_plot = input("Save Plot? [y/n] ") if save_plot == "y": plt.savefig('plot_{}.png'.format(project.custom_use_solver)) diff --git a/examples/tutorial_gridsearch.py b/examples/tutorial_gridsearch.py index bf55ad4..a0d872a 100644 --- a/examples/tutorial_gridsearch.py +++ b/examples/tutorial_gridsearch.py @@ -1,116 +1,131 @@ +# DKFZ +# +# +# Copyright (c) German Cancer Research Center, +# Division of Medical Image Computing. +# All rights reserved. +# +# This software is distributed WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. +# +# See LICENSE +# +# Author: Sven Wanner (s.wanner@dkfz.de) + # In this tutorial we solve an optimization problem using the GridsearchSolver # Gridsearch is very inefficient a Randomsearch might most of the time be the # better choice. # import the HyppopyProject class keeping track of inputs from hyppopy.HyppopyProject import HyppopyProject # import the GridsearchSolver classes from hyppopy.solver.GridsearchSolver import GridsearchSolver # import the Blackboxfunction class wrapping your problem for Hyppopy from hyppopy.BlackboxFunction import BlackboxFunction # To configure the GridsearchSolver we only need the hyperparameter section. Another # difference to the other solvers is that we need to define a gridsampling in addition # to the range: 'data': [0, 1, 100] which means sampling the space from 0 to 1 in 100 # intervals. Gridsearch also supports categorical, uniform, normal and lognormal sampling config = { "hyperparameter": { "C": { "domain": "uniform", "data": [0.0001, 20, 20], "type": "float" }, "gamma": { "domain": "uniform", "data": [0.0001, 20.0, 20], "type": "float" }, "kernel": { "domain": "categorical", "data": ["linear", "sigmoid", "poly", "rbf"], "type": "str" } }, "settings": { "solver": {}, "custom": {} }} # When creating a HyppopyProject instance we # pass the config dictionary to the constructor. project = HyppopyProject(config=config) # Hyppopy offers a class called BlackboxFunction to wrap your problem for Hyppopy. # The function signature is as follows: # BlackboxFunction(blackbox_func=None, # dataloader_func=None, # preprocess_func=None, # callback_func=None, # data=None, # **kwargs) # # Means we can set a couple of function pointers, a data object and an arbitrary number of custom parameter via kwargs. # # - blackbox_func: a function pointer to the actual, user defined, blackbox function that is computing our loss # - dataloader_func: a function pointer to a function handling the dataloading # - preprocess_func: a function pointer to a function automatically executed before starting the optimization process # - callback_func: a function pointer to a function that is called after each iteration with the trail object as input # - data: setting data can be done via dataloader_func or directly # - kwargs are passed to all functions above and thus can be used for parameter sharing between the functions # # (more details see in the documentation) # # Below we demonstrate the usage of all the above by defining a my_dataloader_function which in fact only grabs the # iris dataset from sklearn and returns it. A my_preprocess_function which also does nothing useful here but # demonstrating that a custom parameter can be set via kwargs and used in all of our functions when called within # Hyppopy. The my_callback_function gets as input the dictionary containing the state of the iteration and thus can be # used to access the current state of each solver iteration. Finally we define the actual loss_function # my_loss_function, which gets as input a data object and params. Both parameter are fixed, the first is defined by # the user depending on what is dataloader returns or the data object set in the constructor, the second is a dictionary # with a sample of your hyperparameter space which content is in the choice of the solver. from sklearn.svm import SVC from sklearn.datasets import load_iris from sklearn.model_selection import cross_val_score def my_dataloader_function(**kwargs): print("Dataloading...") iris_data = load_iris() return [iris_data.data, iris_data.target] def my_callback_function(**kwargs): print("\r{}".format(kwargs), end="") def my_loss_function(data, params): clf = SVC(**params) return -cross_val_score(estimator=clf, X=data[0], y=data[1], cv=3).mean() # We now create the BlackboxFunction object and pass all function pointers defined above, # as well as 2 dummy parameter (my_preproc_param, my_dataloader_input) for demonstration purposes. blackbox = BlackboxFunction(blackbox_func=my_loss_function, dataloader_func=my_dataloader_function, callback_func=my_callback_function) # create a solver instance solver = GridsearchSolver(project) # pass the loss function to the solver solver.blackbox = blackbox # run the solver solver.run() # get the result via get_result() which returns a pandas dataframe # containing the complete history and a dict best containing the # best parameter set. df, best = solver.get_results() print("\n") print("*"*100) print("Best Parameter Set:\n{}".format(best)) print("*"*100) diff --git a/examples/tutorial_hyppopyprojectclass.py b/examples/tutorial_hyppopyprojectclass.py index 846a1e5..66d285f 100644 --- a/examples/tutorial_hyppopyprojectclass.py +++ b/examples/tutorial_hyppopyprojectclass.py @@ -1,56 +1,71 @@ +# DKFZ +# +# +# Copyright (c) German Cancer Research Center, +# Division of Medical Image Computing. +# All rights reserved. +# +# This software is distributed WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. +# +# See LICENSE +# +# Author: Sven Wanner (s.wanner@dkfz.de) + # In this tutorial we demonstrate the HyppopyProject class usage # import the HyppopyProject class from hyppopy.HyppopyProject import HyppopyProject # To configure a solver we need to instanciate a HyppopyProject class. # This class can be configured using a nested dict. This dict has two # obligatory sections, hyperparameter and settings. A hyperparameter # is described using a dict containing a section, data and type field # and thus the hyperparameter section is a collection of hyperparameter # dicts. The settings section keeps solver settings. These might depend # on the solver used and need to be checked for each. E.g. Randomsearch, # Hyperopt and Optunity need a solver setting max_iterations, the Grid- # searchSolver don't. config = { "hyperparameter": { "C": { "domain": "uniform", "data": [0.0001, 20], "type": "float" }, "gamma": { "domain": "uniform", "data": [0.0001, 20.0], "type": "float" }, "kernel": { "domain": "categorical", "data": ["linear", "sigmoid", "poly", "rbf"], "type": "str" } }, "settings": { "solver": { "max_iterations": 500 }, "custom": {} }} # When creating a HyppopyProject instance we # pass the config dictionary to the constructor. project = HyppopyProject(config=config) # When building the project programmatically we can also use the methods # add_hyperparameter and add_settings project.clear() project.add_hyperparameter(name="C", domain="uniform", data=[0.0001, 20], dtype="float") project.add_hyperparameter(name="kernel", domain="categorical", data=["linear", "sigmoid"], dtype="str") project.add_settings(section="solver", name="max_iterations", value=500) # The custom section can be used freely project.add_settings(section="custom", name="my_var", value=10) # Settings are automatically transformed to member variables of the project class with the section as prefix if project.solver_max_iterations < 1000 and project.custom_my_var == 10: print("Project configured!") diff --git a/examples/tutorial_multisolver.py b/examples/tutorial_multisolver.py index afda236..1dd4637 100644 --- a/examples/tutorial_multisolver.py +++ b/examples/tutorial_multisolver.py @@ -1,174 +1,189 @@ +# DKFZ +# +# +# Copyright (c) German Cancer Research Center, +# Division of Medical Image Computing. +# All rights reserved. +# +# This software is distributed WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. +# +# See LICENSE +# +# Author: Sven Wanner (s.wanner@dkfz.de) + # In this tutorial we solve an optimization problem using the Hyperopt Solver (http://hyperopt.github.io/hyperopt/). # Hyperopt uses a Baysian - Tree Parzen Estimator - Optimization approach, which means that each iteration computes a # new function value of the blackbox, interpolates a guess for the whole energy function and predicts a point to # compute the next function value at. This next point is not necessarily a "better" value, it's only the value with # the highest uncertainty for the function interpolation. # # See a visual explanation e.g. here (http://philipperemy.github.io/visualization/) # import the HyppopyProject class keeping track of inputs from hyppopy.HyppopyProject import HyppopyProject # import the SolverPool singleton class from hyppopy.SolverPool import SolverPool # import the Blackboxfunction class wrapping your problem for Hyppopy from hyppopy.BlackboxFunction import BlackboxFunction # Next step is defining the problem space and all settings Hyppopy needs to optimize your problem. # The config is a simple nested dictionary with two obligatory main sections, hyperparameter and settings. # The hyperparameter section defines your searchspace. Each hyperparameter is again a dictionary with: # # - a domain ['categorical', 'uniform', 'normal', 'loguniform'] # - the domain data [left bound, right bound] and # - a type of your domain ['str', 'int', 'float'] # # The settings section has two subcategories, solver and custom. The first contains settings for the solver, # here 'max_iterations' - is the maximum number of iteration. # # The custom section allows defining custom parameter. An entry here is transformed to a member variable of the # HyppopyProject class. These can be useful when implementing new solver classes or for control your hyppopy script. # Here we use it as a solver switch to control the usage of our solver via the config. This means with the script # below your can try out every solver by changing use_solver to 'optunity', 'randomsearch', 'gridsearch',... # It can be used like so: project.custom_use_plugin (see below) If using the gridsearch solver, max_iterations is # ignored, instead each hyperparameter must specifiy a number of samples additionally to the range like so: # 'data': [0, 1, 100] which means sampling the space from 0 to 1 in 100 intervals. config = { "hyperparameter": { "C": { "domain": "uniform", "data": [0.0001, 20], "type": "float" }, "gamma": { "domain": "uniform", "data": [0.0001, 20.0], "type": "float" }, "kernel": { "domain": "categorical", "data": ["linear", "sigmoid", "poly", "rbf"], "type": "str" }, "decision_function_shape": { "domain": "categorical", "data": ["ovo", "ovr"], "type": "str" } }, "settings": { "solver": { "max_iterations": 200 }, "custom": { "use_solver": "hyperopt" } }} # When creating a HyppopyProject instance we # pass the config dictionary to the constructor. project = HyppopyProject(config=config) # demonstration of the custom parameter access print("-"*30) print("max_iterations:\t{}".format(project.solver_max_iterations)) print("solver chosen -> {}".format(project.custom_use_solver)) print("-"*30) # The BlackboxFunction signature is as follows: # BlackboxFunction(blackbox_func=None, # dataloader_func=None, # preprocess_func=None, # callback_func=None, # data=None, # **kwargs) # # - blackbox_func: a function pointer to the users loss function # - dataloader_func: a function pointer for handling dataloading. The function is called once before # optimizing. What it returns is passed as first argument to your loss functions # data argument. # - preprocess_func: a function pointer for data preprocessing. The function is called once before # optimizing and gets via kwargs['data'] the raw data object set directly or returned # from dataloader_func. What this function returns is then what is passed as first # argument to your loss function. # - callback_func: a function pointer called after each iteration. The input kwargs is a dictionary # keeping the parameters used in this iteration, the 'iteration' index, the 'loss' # and the 'status'. The function in this example is used for realtime printing it's # input but can also be used for realtime visualization. # - data: if not done via dataloader_func one can set a raw_data object directly # - kwargs: dict that whose content is passed to all functions above. from sklearn.svm import SVC from sklearn.datasets import load_iris from sklearn.model_selection import cross_val_score def my_dataloader_function(**kwargs): print("Dataloading...") # kwargs['params'] allows accessing additional parameter passed, see below my_preproc_param, my_dataloader_input. print("my loading argument: {}".format(kwargs['params']['my_dataloader_input'])) iris_data = load_iris() return [iris_data.data, iris_data.target] def my_preprocess_function(**kwargs): print("Preprocessing...") # kwargs['data'] allows accessing the input data print("data:", kwargs['data'][0].shape, kwargs['data'][1].shape) # kwargs['params'] allows accessing additional parameter passed, see below my_preproc_param, my_dataloader_input. print("kwargs['params']['my_preproc_param']={}".format(kwargs['params']['my_preproc_param']), "\n") # if the preprocessing function returns something, # the input data will be replaced with the data returned by this function. x = kwargs['data'][0] y = kwargs['data'][1] for i in range(x.shape[0]): x[i, :] += kwargs['params']['my_preproc_param'] return [x, y] def my_callback_function(**kwargs): print("\r{}".format(kwargs), end="") def my_loss_function(data, params): clf = SVC(**params) return -cross_val_score(estimator=clf, X=data[0], y=data[1], cv=3).mean() # We now create the BlackboxFunction object and pass all function pointers defined above, # as well as 2 dummy parameter (my_preproc_param, my_dataloader_input) for demonstration purposes. blackbox = BlackboxFunction(blackbox_func=my_loss_function, dataloader_func=my_dataloader_function, preprocess_func=my_preprocess_function, callback_func=my_callback_function, my_preproc_param=1, my_dataloader_input='could/be/a/path') # Last step, is we use our SolverPool which automatically returns the correct solver. # There are multiple ways to get the desired solver from the solver pool. # 1. solver = SolverPool.get('hyperopt') # solver.project = project # 2. solver = SolverPool.get('hyperopt', project) # 3. The SolverPool will look for the field 'use_solver' in the project instance, if # it is present it will be used to specify the solver so that in this case it is enough # to pass the project instance. solver = SolverPool.get(project=project) # Give the solver your blackbox and run it. After execution we can get the result # via get_result() which returns a pandas dataframe containing the complete history # The dict best contains the best parameter set. solver.blackbox = blackbox solver.run() df, best = solver.get_results() print("\n") print("*"*100) print("Best Parameter Set:\n{}".format(best)) print("*"*100) diff --git a/examples/tutorial_simple.py b/examples/tutorial_simple.py index d7e77ef..6e24fae 100644 --- a/examples/tutorial_simple.py +++ b/examples/tutorial_simple.py @@ -1,71 +1,86 @@ +# DKFZ +# +# +# Copyright (c) German Cancer Research Center, +# Division of Medical Image Computing. +# All rights reserved. +# +# This software is distributed WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. +# +# See LICENSE +# +# Author: Sven Wanner (s.wanner@dkfz.de) + # A hyppopy minimal example optimizing a simple demo function f(x,y) = x**2+y**2 # import the HyppopyProject class keeping track of inputs from hyppopy.HyppopyProject import HyppopyProject # import the HyperoptSolver class from hyppopy.solver.HyperoptSolver import HyperoptSolver # To configure the Hyppopy solver we use a simple nested dictionary with two obligatory main sections, # hyperparameter and settings. The hyperparameter section defines your searchspace. Each hyperparameter # is again a dictionary with: # # - a domain ['categorical', 'uniform', 'normal', 'loguniform'] # - the domain data [left bound, right bound] and # - a type of your domain ['str', 'int', 'float'] # # The settings section has two subcategories, solver and custom. The first contains settings for the solver, # here 'max_iterations' - is the maximum number of iteration. # # The custom section allows defining custom parameter. An entry here is transformed to a member variable of the # HyppopyProject class. These can be useful when implementing new solver classes or for control your hyppopy script. # Here we use it as a solver switch to control the usage of our solver via the config. This means with the script # below your can try out every solver by changing use_solver to 'optunity', 'randomsearch', 'gridsearch',... # It can be used like so: project.custom_use_plugin (see below) If using the gridsearch solver, max_iterations is # ignored, instead each hyperparameter must specifiy a number of samples additionally to the range like so: # 'data': [0, 1, 100] which means sampling the space from 0 to 1 in 100 intervals. config = { "hyperparameter": { "x": { "domain": "normal", "data": [-10.0, 10.0], "type": "float" }, "y": { "domain": "uniform", "data": [-10.0, 10.0], "type": "float" } }, "settings": { "solver": { "max_iterations": 500 }, "custom": {} }} # When creating a HyppopyProject instance we # pass the config dictionary to the constructor. project = HyppopyProject(config=config) # The user defined loss function def my_loss_function(x, y): return x**2+y**2 # create a solver instance solver = HyperoptSolver(project) # pass the loss function to the solver solver.blackbox = my_loss_function # run the solver solver.run() df, best = solver.get_results() print("\n") print("*"*100) print("Best Parameter Set:\n{}".format(best)) print("*"*100) diff --git a/hyppopy/BlackboxFunction.py b/hyppopy/BlackboxFunction.py index 0a7bdc6..8fce552 100644 --- a/hyppopy/BlackboxFunction.py +++ b/hyppopy/BlackboxFunction.py @@ -1,98 +1,98 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, -# Division of Medical and Biological Informatics. +# Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # -# See LICENSE.txt or http://www.mitk.org for details. +# See LICENSE # # Author: Sven Wanner (s.wanner@dkfz.de) import os import logging import functools from hyppopy.globals import DEBUGLEVEL LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) def default_kwargs(**defaultKwargs): def actual_decorator(fn): @functools.wraps(fn) def g(*args, **kwargs): defaultKwargs.update(kwargs) return fn(*args, **defaultKwargs) return g return actual_decorator class BlackboxFunction(object): @default_kwargs(blackbox_func=None, dataloader_func=None, preprocess_func=None, callback_func=None, data=None) def __init__(self, **kwargs): self._blackbox_func = None self._preprocess_func = None self._dataloader_func = None self._callback_func = None self._raw_data = None self._data = None self.setup(kwargs) def __call__(self, **kwargs): return self.blackbox_func(self.data, kwargs) def setup(self, kwargs): self._blackbox_func = kwargs['blackbox_func'] self._preprocess_func = kwargs['preprocess_func'] self._dataloader_func = kwargs['dataloader_func'] self._callback_func = kwargs['callback_func'] self._raw_data = kwargs['data'] self._data = self._raw_data del kwargs['blackbox_func'] del kwargs['preprocess_func'] del kwargs['dataloader_func'] del kwargs['data'] params = kwargs if self.dataloader_func is not None: self._raw_data = self.dataloader_func(params=params) assert self._raw_data is not None, "Missing data exception!" assert self.blackbox_func is not None, "Missing blackbox fucntion exception!" if self.preprocess_func is not None: result = self.preprocess_func(data=self._raw_data, params=params) if result is not None: self._data = result else: self._data = self._raw_data else: self._data = self._raw_data @property def blackbox_func(self): return self._blackbox_func @property def preprocess_func(self): return self._preprocess_func @property def dataloader_func(self): return self._dataloader_func @property def callback_func(self): return self._callback_func @property def raw_data(self): return self._raw_data @property def data(self): return self._data diff --git a/hyppopy/HyppopyProject.py b/hyppopy/HyppopyProject.py index 20eb1bc..19409fa 100644 --- a/hyppopy/HyppopyProject.py +++ b/hyppopy/HyppopyProject.py @@ -1,103 +1,103 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, -# Division of Medical and Biological Informatics. +# Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # -# See LICENSE.txt or http://www.mitk.org for details. +# See LICENSE # # Author: Sven Wanner (s.wanner@dkfz.de) import warnings from hyppopy.globals import * LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) class HyppopyProject(object): def __init__(self, config=None): self._hyperparameter = {} self._settings = {} self._extmembers = [] if config is not None: self.set_config(config) def clear(self): self._hyperparameter = {} self._settings = {} for added in self._extmembers: if added in self.__dict__.keys(): del self.__dict__[added] self._extmembers = [] def set_config(self, config): self.clear() assert isinstance(config, dict), "Input Error, config of type {} not supported!".format(type(config)) assert HYPERPARAMETERPATH in config.keys(), "Missing hyperparameter section in config dict" #assert SETTINGSPATH in config.keys(), "Missing settings section in config dict" if not SETTINGSPATH in config.keys(): config[SETTINGSPATH] = {"solver": {"max_iterations": DEFAULTITERATIONS}} msg = "config dict had no section {0}/solver/max_iterations, set default value: {1}".format(SETTINGSPATH, DEFAULTITERATIONS) warnings.warn(msg) LOG.warning(msg) self._hyperparameter = config[HYPERPARAMETERPATH] self._settings = config[SETTINGSPATH] self.parse_members() def add_hyperparameter(self, name, domain, data, dtype): assert isinstance(name, str), "precondition violation, name of type {} not allowed, expect str!".format(type(name)) assert isinstance(domain, str), "precondition violation, domain of type {} not allowed, expect str!".format(type(domain)) assert domain in SUPPORTED_DOMAINS, "domain {} not supported, expect {}!".format(domain, SUPPORTED_DOMAINS) assert isinstance(data, list) or isinstance(data, tuple), "precondition violation, data of type {} not allowed, expect list or tuple!".format(type(data)) if domain != "categorical": assert len(data) == 3 or len(data) == 2, "precondition violation, data must be a list of len 2 or 3" assert isinstance(dtype, str), "precondition violation, dtype of type {} not allowed, expect str!".format(type(dtype)) assert dtype in SUPPORTED_DTYPES, "precondition violation, dtype {} not supported, expect {}!".format(dtype, SUPPORTED_DTYPES) self._hyperparameter[name] = {"domain": domain, "data": data, "type": dtype} def add_settings(self, section, name, value): assert isinstance(section, str), "precondition violation, section of type {} not allowed, expect str!".format(type(section)) assert isinstance(name, str), "precondition violation, name of type {} not allowed, expect str!".format(type(name)) if section not in self._settings.keys(): self._settings[section] = {} self._settings[section][name] = value self.parse_members() def parse_members(self): for section_name, content in self.settings.items(): for name, value in content.items(): member_name = section_name + "_" + name if member_name not in self._extmembers: setattr(self, member_name, value) self._extmembers.append(member_name) else: self.__dict__[member_name] = value def get_typeof(self, hyperparametername): if not hyperparametername in self.hyperparameter.keys(): return None dtype = self.hyperparameter[hyperparametername]["type"] if dtype == 'str': return str if dtype == 'int': return int if dtype == 'float': return float @property def hyperparameter(self): return self._hyperparameter @property def settings(self): return self._settings diff --git a/hyppopy/projectmanager.py b/hyppopy/ProjectManager.py similarity index 94% rename from hyppopy/projectmanager.py rename to hyppopy/ProjectManager.py index 79fe26c..4257bf8 100644 --- a/hyppopy/projectmanager.py +++ b/hyppopy/ProjectManager.py @@ -1,69 +1,69 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, -# Division of Medical and Biological Informatics. +# Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # -# See LICENSE.txt or http://www.mitk.org for details. +# See LICENSE # # Author: Sven Wanner (s.wanner@dkfz.de) from .Singleton import * import os import logging from hyppopy.HyppopyProject import HyppopyProject from hyppopy.globals import DEBUGLEVEL LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) @singleton_object class ProjectManager(metaclass=Singleton): def __init__(self): self._current_project = None self._projects = {} def clear_all(self): pass def new_project(self, name="HyppopyProject", config=None): if name in self._projects.keys(): name = self.check_projectname(name) self._projects[name] = HyppopyProject(config) self._current_project = self._projects[name] return self._current_project def check_projectname(self, name): split = name.split(".") if len(split) == 0: return split[0] + "." + str(0).zfill(3) else: try: number = int(split[-1]) del split[-1] except: number = 0 return '.'.join(split) + "." + str(number).zfill(3) def get_current(self): if self._current_project is None: self.new_project() return self._current_project def get_project(self, name): if name in self._projects.keys(): self._current_project = self._projects[name] return self.get_current() return self.new_project(name) def get_projectnames(self): return self._projects.keys() diff --git a/hyppopy/Singleton.py b/hyppopy/Singleton.py index 6f61770..90575de 100644 --- a/hyppopy/Singleton.py +++ b/hyppopy/Singleton.py @@ -1,52 +1,52 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, -# Division of Medical and Biological Informatics. +# Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # -# See LICENSE.txt or http://www.mitk.org for details. +# See LICENSE # # Author: Sven Wanner (s.wanner@dkfz.de) class Singleton(type): _instances = {} def __call__(cls, *args, **kwargs): if cls not in cls._instances: cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) return cls._instances[cls] @classmethod def __instancecheck__(mcs, instance): if instance.__class__ is mcs: return True else: return isinstance(instance.__class__, mcs) def singleton_object(cls): """Class decorator that transforms (and replaces) a class definition (which must have a Singleton metaclass) with the actual singleton object. Ensures that the resulting object can still be "instantiated" (i.e., called), returning the same object. Also ensures the object can be pickled, is hashable, and has the correct string representation (the name of the singleton) """ assert isinstance(cls, Singleton), cls.__name__ + " must use Singleton metaclass" def self_instantiate(self): return self cls.__call__ = self_instantiate cls.__hash__ = lambda self: hash(cls) cls.__repr__ = lambda self: cls.__name__ cls.__reduce__ = lambda self: cls.__name__ obj = cls() obj.__name__ = cls.__name__ return obj diff --git a/hyppopy/Solver/BayesOptSolver.py b/hyppopy/Solver/BayesOptSolver.py index bed6e7a..4179539 100644 --- a/hyppopy/Solver/BayesOptSolver.py +++ b/hyppopy/Solver/BayesOptSolver.py @@ -1,84 +1,84 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, -# Division of Medical and Biological Informatics. +# Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # -# See LICENSE.txt or http://www.mitk.org for details. +# See LICENSE # # Author: Sven Wanner (s.wanner@dkfz.de) import os import logging import warnings import numpy as np from pprint import pformat from hyperopt import Trials from bayes_opt import BayesianOptimization from hyppopy.globals import DEBUGLEVEL from hyppopy.solver.HyppopySolver import HyppopySolver LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) class BayesOptSolver(HyppopySolver): def __init__(self, project=None): HyppopySolver.__init__(self, project) self._searchspace = None self._idx = None def reformat_parameter(self, params): out_params = {} for name, value in params.items(): if self._searchspace[name]["domain"] == "categorical": out_params[name] = self._searchspace[name]["data"][int(np.round(value))] else: if self._searchspace[name]["type"] == "int": out_params[name] = int(np.round(value)) else: out_params[name] = value return out_params def loss_function_call(self, trial, params): params = self.reformat_parameter(params) for key in params.keys(): if self.project.get_typeof(key) is int: params[key] = int(round(params[key])) return self.blackbox(**params) def execute_solver(self, searchspace): LOG.debug("execute_solver using solution space:\n\n\t{}\n".format(pformat(searchspace))) self.trials = Trials() self._idx = 0 try: optimizer = BayesianOptimization(f=self.loss_function, pbounds=searchspace, verbose=0) optimizer.maximize(init_points=2, n_iter=self.max_iterations) self.best = self.reformat_parameter(optimizer.max["params"]) except Exception as e: LOG.error("internal error in bayes_opt maximize occured. {}".format(e)) raise BrokenPipeError("internal error in bayes_opt maximize occured. {}".format(e)) def convert_searchspace(self, hyperparameter): LOG.debug("convert input parameter\n\n\t{}\n".format(pformat(hyperparameter))) self._searchspace = hyperparameter pbounds = {} for name, param in hyperparameter.items(): if param["domain"] != "categorical": if param["domain"] != "uniform": msg = "Warning: BayesOpt cannot handle {} domain. Only uniform and categorical domains are supported!".format( param["domain"]) warnings.warn(msg) LOG.warning(msg) pbounds[name] = (param["data"][0], param["data"][1]) else: pbounds[name] = (0, len(param["data"])-1) return pbounds diff --git a/hyppopy/Solver/GridsearchSolver.py b/hyppopy/Solver/GridsearchSolver.py index 3946ca4..920c33f 100644 --- a/hyppopy/Solver/GridsearchSolver.py +++ b/hyppopy/Solver/GridsearchSolver.py @@ -1,182 +1,182 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, -# Division of Medical and Biological Informatics. +# Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # -# See LICENSE.txt or http://www.mitk.org for details. +# See LICENSE # # Author: Sven Wanner (s.wanner@dkfz.de) import os import logging import numpy as np from pprint import pformat from scipy.stats import norm from itertools import product from hyppopy.globals import DEBUGLEVEL from hyppopy.solver.HyppopySolver import HyppopySolver LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) def get_uniform_axis_sample(a, b, N, dtype): """ returns a uniform sample x(n) in the range [a,b] sampled at N pojnts :param a: left value range bound :param b: right value range bound :param N: discretization of intervall [a,b] :param dtype: data type :return: [list] axis range """ assert a < b, "condition a < b violated!" assert isinstance(N, int), "condition N of type int violated!" assert isinstance(dtype, str), "condition type of type str violated!" if dtype == "int": return list(np.linspace(a, b, N).astype(int)) elif dtype == "float" or dtype == "double": return list(np.linspace(a, b, N)) else: raise AssertionError("dtype {} not supported for uniform sampling!".format(dtype)) def get_norm_cdf(N): """ returns a normed gaussian cdf (range [0,1]) with N sampling points :param N: sampling points :return: [ndarray] gaussian cdf function values """ assert isinstance(N, int), "condition N of type int violated!" even = True if N % 2 != 0: N -= 1 even = False N = int(N/2) sigma = 1/3 x = np.linspace(0, 1, N) y1 = norm.cdf(x, loc=0, scale=sigma)-0.5 if not even: y1 = np.append(y1, [0.5]) y2 = 1-(norm.cdf(x, loc=0, scale=sigma)-0.5) y2 = np.flip(y2, axis=0) y = np.concatenate((y1, y2), axis=0) return y def get_gaussian_axis_sample(a, b, N, dtype): """ returns a function value f(n) where f is a gaussian cdf in range [a, b] and N sampling points :param a: left value range bound :param b: right value range bound :param N: discretization of intervall [a,b] :param dtype: data type :return: [list] axis range """ assert a < b, "condition a < b violated!" assert isinstance(N, int), "condition N of type int violated!" assert isinstance(dtype, str), "condition type of type str violated!" data = [] for n in range(N): x = a + get_norm_cdf(N)[n]*(b-a) if dtype == "int": data.append(int(x)) elif dtype == "float" or dtype == "double": data.append(x) else: raise AssertionError("dtype {} not supported for uniform sampling!".format(dtype)) return data def get_logarithmic_axis_sample(a, b, N, dtype): """ returns a function value f(n) where f is logarithmic function e^x sampling the exponent range [log(a), log(b)] linear at N sampling points. The function values returned are in the range [a, b]. :param a: left value range bound :param b: right value range bound :param N: discretization of intervall [a,b] :param dtype: data type :return: [list] axis range """ assert a < b, "condition a < b violated!" assert a > 0, "condition a > 0 violated!" assert isinstance(N, int), "condition N of type int violated!" assert isinstance(dtype, str), "condition type of type str violated!" # convert input range into exponent range lexp = np.log(a) rexp = np.log(b) exp_range = np.linspace(lexp, rexp, N) data = [] for n in range(exp_range.shape[0]): x = np.exp(exp_range[n]) if dtype == "int": data.append(int(x)) elif dtype == "float" or dtype == "double": data.append(x) else: raise AssertionError("dtype {} not supported for uniform sampling!".format(dtype)) return data class GridsearchSolver(HyppopySolver): """ The GridsearchSolver class implements a gridsearch optimization. The gridsearch supports categorical, uniform, normal and loguniform sampling. To use the GridsearchSolver, besides a range, one must specifiy the number of samples in the domain, e.g. 'data': [0, 1, 100] """ def __init__(self, project=None): HyppopySolver.__init__(self, project) self._has_maxiteration_field = False def loss_function_call(self, trial, params): loss = self.blackbox(**params) if loss is None: return np.nan return loss def execute_solver(self, searchspace): for x in product(*searchspace[1]): params = {} for name, value in zip(searchspace[0], x): params[name] = value try: self.loss_function(**params) except Exception as e: msg = "internal error in randomsearch execute_solver occured. {}".format(e) LOG.error(msg) raise BrokenPipeError(msg) self.best = self._trials.argmin def convert_searchspace(self, hyperparameter): """ the function converts the standard parameter input into a range list depending on the domain. These rangelists are later used with itertools product to create a paramater space sample of each combination. :param hyperparameter: [dict] hyperparameter space :return: [list] name and range for each parameter space axis """ LOG.debug("convert input parameter\n\n\t{}\n".format(pformat(hyperparameter))) searchspace = [[], []] for name, param in hyperparameter.items(): if param["domain"] == "categorical": searchspace[0].append(name) searchspace[1].append(param["data"]) elif param["domain"] == "uniform": searchspace[0].append(name) searchspace[1].append(get_uniform_axis_sample(param["data"][0], param["data"][1], param["data"][2], param["type"])) elif param["domain"] == "normal": searchspace[0].append(name) searchspace[1].append(get_gaussian_axis_sample(param["data"][0], param["data"][1], param["data"][2], param["type"])) elif param["domain"] == "loguniform": searchspace[0].append(name) searchspace[1].append(get_logarithmic_axis_sample(param["data"][0], param["data"][1], param["data"][2], param["type"])) return searchspace diff --git a/hyppopy/Solver/HyperoptSolver.py b/hyppopy/Solver/HyperoptSolver.py index 0f304b2..b9b91ff 100644 --- a/hyppopy/Solver/HyperoptSolver.py +++ b/hyppopy/Solver/HyperoptSolver.py @@ -1,158 +1,158 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, -# Division of Medical and Biological Informatics. +# Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # -# See LICENSE.txt or http://www.mitk.org for details. +# See LICENSE # # Author: Sven Wanner (s.wanner@dkfz.de) import os import copy import logging import numpy as np from pprint import pformat from hyperopt import fmin, tpe, hp, STATUS_OK, STATUS_FAIL, Trials from hyppopy.globals import DEBUGLEVEL from hyppopy.solver.HyppopySolver import HyppopySolver from hyppopy.BlackboxFunction import BlackboxFunction LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) class HyperoptSolver(HyppopySolver): def __init__(self, project=None): HyppopySolver.__init__(self, project) self._searchspace = None def loss_function(self, params): for name, p in self._searchspace.items(): if p["domain"] != "categorical": if params[name] < p["data"][0]: params[name] = p["data"][0] if params[name] > p["data"][1]: params[name] = p["data"][1] status = STATUS_FAIL try: loss = self.blackbox(**params) if loss is not None: status = STATUS_OK else: loss = 1e9 except Exception as e: LOG.error("execution of self.blackbox(**params) failed due to:\n {}".format(e)) status = STATUS_FAIL loss = 1e9 if isinstance(self.blackbox, BlackboxFunction) and self.blackbox.callback_func is not None: cbd = copy.deepcopy(params) cbd['iterations'] = self._trials.trials[-1]['tid'] + 1 cbd['loss'] = loss cbd['status'] = status self.blackbox.callback_func(**cbd) return {'loss': loss, 'status': status} def execute_solver(self, searchspace): LOG.debug("execute_solver using solution space:\n\n\t{}\n".format(pformat(searchspace))) self.trials = Trials() try: self.best = fmin(fn=self.loss_function, space=searchspace, algo=tpe.suggest, max_evals=self.max_iterations, trials=self.trials) except Exception as e: msg = "internal error in hyperopt.fmin occured. {}".format(e) LOG.error(msg) raise BrokenPipeError(msg) def convert_searchspace(self, hyperparameter): self._searchspace = hyperparameter solution_space = {} for name, content in hyperparameter.items(): param_settings = {'name': name} for key, value in content.items(): if key == 'domain': param_settings['domain'] = value elif key == 'data': param_settings['data'] = value elif key == 'type': param_settings['dtype'] = value solution_space[name] = self.convert(param_settings) return solution_space def convert(self, param_settings): name = param_settings["name"] domain = param_settings["domain"] dtype = param_settings["dtype"] data = param_settings["data"] assert isinstance(data, list), "precondition violation. data of type {} not allowed!".format(type(data)) assert len(data) >= 2, "precondition violation, data must be of length 2, [left_bound, right_bound]" assert isinstance(domain, str), "precondition violation. domain of type {} not allowed!".format(type(domain)) assert isinstance(dtype, str), "precondition violation. dtype of type {} not allowed!".format(type(dtype)) if domain == "uniform": if dtype == "float" or dtype == "double": return hp.uniform(name, data[0], data[1]) elif dtype == "int": data = list(np.arange(int(data[0]), int(data[1] + 1))) return hp.choice(name, data) else: msg = "cannot convert the type {} in domain {}".format(dtype, domain) LOG.error(msg) raise LookupError(msg) elif domain == "loguniform": if dtype == "float" or dtype == "double": if data[0] == 0: data[0] += 1e-23 assert data[0] > 0, "precondition Violation, a < 0!" assert data[0] < data[1], "precondition Violation, a > b!" assert data[1] > 0, "precondition Violation, b < 0!" lexp = np.log(data[0]) rexp = np.log(data[1]) assert lexp is not np.nan, "precondition violation, left bound input error, results in nan!" assert rexp is not np.nan, "precondition violation, right bound input error, results in nan!" return hp.loguniform(name, lexp, rexp) else: msg = "cannot convert the type {} in domain {}".format(dtype, domain) LOG.error(msg) raise LookupError(msg) elif domain == "normal": if dtype == "float" or dtype == "double": mu = (data[1] - data[0]) / 2.0 sigma = mu / 3 return hp.normal(name, data[0] + mu, sigma) else: msg = "cannot convert the type {} in domain {}".format(dtype, domain) LOG.error(msg) raise LookupError(msg) elif domain == "categorical": if dtype == 'str': return hp.choice(name, data) elif dtype == 'bool': data = [] for elem in data: if elem == "true" or elem == "True" or elem == 1 or elem == "1": data.append(True) elif elem == "false" or elem == "False" or elem == 0 or elem == "0": data.append(False) else: msg = "cannot convert the type {} in domain {}, unknown bool type value".format(dtype, domain) LOG.error(msg) raise LookupError(msg) return hp.choice(name, data) else: msg = "Precondition violation, domain named {} not available!".format(domain) LOG.error(msg) raise IOError(msg) diff --git a/hyppopy/Solver/HyppopySolver.py b/hyppopy/Solver/HyppopySolver.py index bc08ddb..07d3808 100644 --- a/hyppopy/Solver/HyppopySolver.py +++ b/hyppopy/Solver/HyppopySolver.py @@ -1,278 +1,287 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, -# Division of Medical and Biological Informatics. +# Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # -# See LICENSE.txt or http://www.mitk.org for details. +# See LICENSE # # Author: Sven Wanner (s.wanner@dkfz.de) import abc import os import copy import types import logging import datetime import numpy as np import pandas as pd from hyperopt import Trials from hyppopy.globals import DEBUGLEVEL from hyppopy.HyppopyProject import HyppopyProject from hyppopy.BlackboxFunction import BlackboxFunction from hyppopy.VirtualFunction import VirtualFunction from hyppopy.globals import DEBUGLEVEL, DEFAULTITERATIONS LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) class HyppopySolver(object): - + """ + The HyppopySolver class is the base class for all solver addons. It defines virtual functions a child class has + to implement to deal with the front-end communication, orchestrating the optimization process and ensuring a proper + process information storing. + The key idea is that the HyppopySolver class defines an interface to configure and run an object instance of itself + independently from the concrete solver lib used to optimize in the background. To achieve this goal an addon + developer needs to implement the abstract methods 'convert_searchspace', 'execute_solver' and 'loss_function_call'. + These methods abstract the peculiarities of the solver libs to offer, on the user side, a simple and consistent + parameter space configuration and optimization procedure. The method 'convert_searchspace' + """ def __init__(self, project=None): self._idx = None self._best = None self._trials = None self._blackbox = None self._max_iterations = None self._project = project self._total_duration = None self._solver_overhead = None self._time_per_iteration = None self._accumulated_blackbox_time = None self._has_maxiteration_field = True @abc.abstractmethod def execute_solver(self, searchspace): raise NotImplementedError('users must define execute_solver to use this class') @abc.abstractmethod def convert_searchspace(self, hyperparameter): raise NotImplementedError('users must define convert_searchspace to use this class') @abc.abstractmethod def loss_function_call(self): raise NotImplementedError('users must define convert_searchspace to use this class') def loss_function(self, **params): self._idx += 1 vals = {} idx = {} for key, value in params.items(): vals[key] = [value] idx[key] = [self._idx] trial = {'tid': self._idx, 'result': {'loss': None, 'status': 'ok'}, 'misc': { 'tid': self._idx, 'idxs': idx, 'vals': vals }, 'book_time': datetime.datetime.now(), 'refresh_time': None } try: loss = self.loss_function_call(trial, params) trial['result']['loss'] = loss trial['result']['status'] = 'ok' if loss == np.nan: trial['result']['status'] = 'failed' except Exception as e: LOG.error("computing loss failed due to:\n {}".format(e)) loss = np.nan trial['result']['loss'] = np.nan trial['result']['status'] = 'failed' trial['refresh_time'] = datetime.datetime.now() self._trials.trials.append(trial) if isinstance(self.blackbox, BlackboxFunction) and self.blackbox.callback_func is not None: cbd = copy.deepcopy(params) cbd['iterations'] = self._idx cbd['loss'] = loss cbd['status'] = trial['result']['status'] self.blackbox.callback_func(**cbd) return loss def run(self, print_stats=True): self._idx = 0 self.trials = Trials() if self._has_maxiteration_field: if 'solver_max_iterations' not in self.project.__dict__: msg = "Missing max_iteration entry in project, use default {}!".format(DEFAULTITERATIONS) LOG.warning(msg) print("WARNING: {}".format(msg)) setattr(self.project, 'solver_max_iterations', DEFAULTITERATIONS) self._max_iterations = self.project.solver_max_iterations start_time = datetime.datetime.now() try: search_space = self.convert_searchspace(self.project.hyperparameter) except Exception as e: msg = "Failed to convert searchspace, error: {}".format(e) LOG.error(msg) raise AssertionError(msg) try: self.execute_solver(search_space) except Exception as e: msg = "Failed to execute solver, error: {}".format(e) LOG.error(msg) raise AssertionError(msg) end_time = datetime.datetime.now() dt = end_time - start_time days = divmod(dt.total_seconds(), 86400) hours = divmod(days[1], 3600) minutes = divmod(hours[1], 60) seconds = divmod(minutes[1], 1) milliseconds = divmod(seconds[1], 0.001) self._total_duration = [int(days[0]), int(hours[0]), int(minutes[0]), int(seconds[0]), int(milliseconds[0])] if print_stats: self.print_best() self.print_timestats() def get_results(self): results = {'duration': [], 'losses': []} pset = self.trials.trials[0]['misc']['vals'] for p in pset.keys(): results[p] = [] for n, trial in enumerate(self.trials.trials): t1 = trial['book_time'] t2 = trial['refresh_time'] results['duration'].append((t2 - t1).microseconds / 1000.0) results['losses'].append(trial['result']['loss']) losses = np.array(results['losses']) results['losses'] = list(losses) pset = trial['misc']['vals'] for p in pset.items(): results[p[0]].append(p[1][0]) return pd.DataFrame.from_dict(results), self.best def print_best(self): print("\n") print("#" * 40) print("### Best Parameter Choice ###") print("#" * 40) for name, value in self.best.items(): print(" - {}\t:\t{}".format(name, value)) print("\n - number of iterations\t:\t{}".format(self.trials.trials[-1]['tid']+1)) print(" - total time\t:\t{}d:{}h:{}m:{}s:{}ms".format(self._total_duration[0], self._total_duration[1], self._total_duration[2], self._total_duration[3], self._total_duration[4])) print("#" * 40) def compute_time_statistics(self): dts = [] for trial in self._trials.trials: if 'book_time' in trial.keys() and 'refresh_time' in trial.keys(): dt = trial['refresh_time'] - trial['book_time'] dts.append(dt.total_seconds()) self._time_per_iteration = np.mean(dts) * 1e3 self._accumulated_blackbox_time = np.sum(dts) * 1e3 tmp = self.total_duration - self._accumulated_blackbox_time self._solver_overhead = int(np.round(100.0 / (self.total_duration+1e-12) * tmp)) def print_timestats(self): print("\n") print("#" * 40) print("### Timing Statistics ###") print("#" * 40) print(" - per iteration: {}ms".format(int(self.time_per_iteration*1e4)/10000)) print(" - total time: {}d:{}h:{}m:{}s:{}ms".format(self._total_duration[0], self._total_duration[1], self._total_duration[2], self._total_duration[3], self._total_duration[4])) print("#" * 40) print(" - solver overhead: {}%".format(self.solver_overhead)) @property def project(self): return self._project @project.setter def project(self, value): if not isinstance(value, HyppopyProject): msg = "Input error, project_manager of type: {} not allowed!".format(type(value)) LOG.error(msg) raise IOError(msg) self._project = value @property def blackbox(self): return self._blackbox @blackbox.setter def blackbox(self, value): if isinstance(value, types.FunctionType) or isinstance(value, BlackboxFunction) or isinstance(value, VirtualFunction): self._blackbox = value else: self._blackbox = None msg = "Input error, blackbox of type: {} not allowed!".format(type(value)) LOG.error(msg) raise IOError(msg) @property def best(self): return self._best @best.setter def best(self, value): if not isinstance(value, dict): msg = "Input error, best of type: {} not allowed!".format(type(value)) LOG.error(msg) raise IOError(msg) self._best = value @property def trials(self): return self._trials @trials.setter def trials(self, value): self._trials = value @property def max_iterations(self): return self._max_iterations @max_iterations.setter def max_iterations(self, value): if not isinstance(value, int): msg = "Input error, max_iterations of type: {} not allowed!".format(type(value)) LOG.error(msg) raise IOError(msg) if value < 1: msg = "Precondition violation, max_iterations < 1!" LOG.error(msg) raise IOError(msg) self._max_iterations = value @property def total_duration(self): return (self._total_duration[0]*86400 + self._total_duration[1] * 3600 + self._total_duration[2] * 60 + self._total_duration[3]) * 1000 + self._total_duration[4] @property def solver_overhead(self): if self._solver_overhead is None: self.compute_time_statistics() return self._solver_overhead @property def time_per_iteration(self): if self._time_per_iteration is None: self.compute_time_statistics() return self._time_per_iteration @property def accumulated_blackbox_time(self): if self._accumulated_blackbox_time is None: self.compute_time_statistics() return self._accumulated_blackbox_time diff --git a/hyppopy/Solver/OptunaSolver.py b/hyppopy/Solver/OptunaSolver.py index dcbcaf8..6c848fc 100644 --- a/hyppopy/Solver/OptunaSolver.py +++ b/hyppopy/Solver/OptunaSolver.py @@ -1,82 +1,82 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, -# Division of Medical and Biological Informatics. +# Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # -# See LICENSE.txt or http://www.mitk.org for details. +# See LICENSE # # Author: Sven Wanner (s.wanner@dkfz.de) import os import optuna import logging import warnings import numpy as np from pprint import pformat from hyppopy.globals import DEBUGLEVEL from hyppopy.solver.HyppopySolver import HyppopySolver LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) class OptunaSolver(HyppopySolver): def __init__(self, project=None): HyppopySolver.__init__(self, project) self._searchspace = None def reformat_parameter(self, params): out_params = {} for name, value in params.items(): if self._searchspace[name]["domain"] == "categorical": out_params[name] = self._searchspace[name]["data"][int(np.round(value))] else: if self._searchspace[name]["type"] == "int": out_params[name] = int(np.round(value)) else: out_params[name] = value return out_params def trial_cache(self, trial): params = {} for name, param in self._searchspace.items(): if param["domain"] == "categorical": params[name] = trial.suggest_categorical(name, param["data"]) else: params[name] = trial.suggest_uniform(name, param["data"][0], param["data"][1]) return self.loss_function(**params) def loss_function_call(self, trial, params): for key in params.keys(): if self.project.get_typeof(key) is int: params[key] = int(round(params[key])) return self.blackbox(**params) def execute_solver(self, searchspace): LOG.debug("execute_solver using solution space:\n\n\t{}\n".format(pformat(searchspace))) self._searchspace = searchspace try: study = optuna.create_study() study.optimize(self.trial_cache, n_trials=self.max_iterations) self.best = study.best_trial.params except Exception as e: LOG.error("internal error in bayes_opt maximize occured. {}".format(e)) raise BrokenPipeError("internal error in bayes_opt maximize occured. {}".format(e)) def convert_searchspace(self, hyperparameter): LOG.debug("convert input parameter\n\n\t{}\n".format(pformat(hyperparameter))) for name, param in hyperparameter.items(): if param["domain"] != "categorical" and param["domain"] != "uniform": msg = "Warning: Optuna cannot handle {} domain. Only uniform and categorical domains are supported!".format(param["domain"]) warnings.warn(msg) LOG.warning(msg) return hyperparameter diff --git a/hyppopy/Solver/OptunitySolver.py b/hyppopy/Solver/OptunitySolver.py index 379ea7d..b4611ce 100644 --- a/hyppopy/Solver/OptunitySolver.py +++ b/hyppopy/Solver/OptunitySolver.py @@ -1,98 +1,98 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, -# Division of Medical and Biological Informatics. +# Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # -# See LICENSE.txt or http://www.mitk.org for details. +# See LICENSE # # Author: Sven Wanner (s.wanner@dkfz.de) import os import logging import optunity import warnings from pprint import pformat from hyppopy.globals import DEBUGLEVEL LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) from hyppopy.solver.HyppopySolver import HyppopySolver class OptunitySolver(HyppopySolver): def __init__(self, project=None): HyppopySolver.__init__(self, project) self._solver_info = None self.opt_trials = None def loss_function_call(self, trial, params): for key in params.keys(): if self.project.get_typeof(key) is int: params[key] = int(round(params[key])) return self.blackbox(**params) def execute_solver(self, searchspace): LOG.debug("execute_solver using solution space:\n\n\t{}\n".format(pformat(searchspace))) try: self.best, _, _ = optunity.minimize_structured(f=self.loss_function, num_evals=self.max_iterations, search_space=searchspace) except Exception as e: LOG.error("internal error in optunity.minimize_structured occured. {}".format(e)) raise BrokenPipeError("internal error in optunity.minimize_structured occured. {}".format(e)) def split_categorical(self, pdict): categorical = {} uniform = {} for name, pset in pdict.items(): for key, value in pset.items(): if key == 'domain' and value == 'categorical': categorical[name] = pset elif key == 'domain': if value != 'uniform': msg = "Warning: Optunity cannot handle {} domain. Only uniform and categorical domains are supported!".format(value) warnings.warn(msg) LOG.warning(msg) uniform[name] = pset return categorical, uniform def convert_searchspace(self, hyperparameter): LOG.debug("convert input parameter\n\n\t{}\n".format(pformat(hyperparameter))) solution_space = {} # split input in categorical and non-categorical data cat, uni = self.split_categorical(hyperparameter) # build up dictionary keeping all non-categorical data uniforms = {} for key, value in uni.items(): for key2, value2 in value.items(): if key2 == 'data': if len(value2) == 3: uniforms[key] = value2[0:2] elif len(value2) == 2: uniforms[key] = value2 else: raise AssertionError("precondition violation, optunity searchspace needs list with left and right range bounds!") if len(cat) == 0: return uniforms # build nested categorical structure inner_level = uniforms for key, value in cat.items(): tmp = {} tmp2 = {} for key2, value2 in value.items(): if key2 == 'data': for elem in value2: tmp[elem] = inner_level tmp2[key] = tmp inner_level = tmp2 solution_space = tmp2 return solution_space diff --git a/hyppopy/Solver/RandomsearchSolver.py b/hyppopy/Solver/RandomsearchSolver.py index 5cd3aed..fff9b18 100644 --- a/hyppopy/Solver/RandomsearchSolver.py +++ b/hyppopy/Solver/RandomsearchSolver.py @@ -1,159 +1,159 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, -# Division of Medical and Biological Informatics. +# Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # -# See LICENSE.txt or http://www.mitk.org for details. +# See LICENSE # # Author: Sven Wanner (s.wanner@dkfz.de) import os import copy import random import logging import numpy as np from pprint import pformat from hyppopy.globals import DEBUGLEVEL from hyppopy.solver.HyppopySolver import HyppopySolver LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) def draw_uniform_sample(param): """ function draws a random sample from a uniform range :param param: [dict] input hyperparameter discription :return: random sample value of type data['type'] """ assert param['type'] != 'str', "cannot sample a string list!" assert param['data'][0] < param['data'][1], "precondition violation: data[0] > data[1]!" s = random.random() s *= np.abs(param['data'][1] - param['data'][0]) s += param['data'][0] if param['type'] == 'int': s = int(np.round(s)) if s < param['data'][0]: s = int(param['data'][0]) if s > param['data'][1]: s = int(param['data'][1]) return s def draw_normal_sample(param): """ function draws a random sample from a normal distributed range :param param: [dict] input hyperparameter discription :return: random sample value of type data['type'] """ assert param['type'] != 'str', "cannot sample a string list!" assert param['data'][0] < param['data'][1], "precondition violation: data[0] > data[1]!" mu = (param['data'][1] - param['data'][0]) / 2 sigma = mu / 3 s = np.random.normal(loc=param['data'][0] + mu, scale=sigma) if s > param['data'][1]: s = param['data'][1] if s < param['data'][0]: s = param['data'][0] s = float(s) if param["type"] == "int": s = int(np.round(s)) return s def draw_loguniform_sample(param): """ function draws a random sample from a logarithmic distributed range :param param: [dict] input hyperparameter discription :return: random sample value of type data['type'] """ assert param['type'] != 'str', "cannot sample a string list!" assert param['data'][0] < param['data'][1], "precondition violation: data[0] > data[1]!" p = copy.deepcopy(param) p['data'][0] = np.log(param['data'][0]) p['data'][1] = np.log(param['data'][1]) assert p['data'][0] is not np.nan, "Precondition violation, left bound input error, results in nan!" assert p['data'][1] is not np.nan, "Precondition violation, right bound input error, results in nan!" x = draw_uniform_sample(p) s = np.exp(x) if s > param['data'][1]: s = param['data'][1] if s < param['data'][0]: s = param['data'][0] return s def draw_categorical_sample(param): """ function draws a random sample from a categorical list :param param: [dict] input hyperparameter discription :return: random sample value of type data['type'] """ return random.sample(param['data'], 1)[0] def draw_sample(param): """ function draws a sample from the input hyperparameter descriptor depending on it's domain :param param: [dict] input hyperparameter discription :return: random sample value of type data['type'] """ assert isinstance(param, dict), "input error, hyperparam descriptors of type {} not allowed!".format(type(param)) assert 'domain' in param.keys(), "input error, hyperparam descriptors need a domain key!" assert 'data' in param.keys(), "input error, hyperparam descriptors need a data key!" assert 'type' in param.keys(), "input error, hyperparam descriptors need a type key!" if param['domain'] == "uniform": return draw_uniform_sample(param) elif param['domain'] == "normal": return draw_normal_sample(param) elif param['domain'] == "loguniform": return draw_loguniform_sample(param) elif param['domain'] == "categorical": return draw_categorical_sample(param) else: raise LookupError("Unknown domain {}".format(param['domain'])) class RandomsearchSolver(HyppopySolver): """ The RandomsearchSolver class implements a randomsearch optimization. The randomsearch supports categorical, uniform, normal and loguniform sampling. The solver draws an independent sample from the parameter space each iteration.""" def __init__(self, project=None): HyppopySolver.__init__(self, project) def loss_function_call(self, trial, params): loss = self.blackbox(**params) if loss is None: return np.nan return loss def execute_solver(self, searchspace): N = self.max_iterations try: for n in range(N): params = {} for name, p in searchspace.items(): params[name] = draw_sample(p) self.loss_function(**params) except Exception as e: msg = "internal error in randomsearch execute_solver occured. {}".format(e) LOG.error(msg) raise BrokenPipeError(msg) self.best = self._trials.argmin def convert_searchspace(self, hyperparameter): """ this function simply pipes the input parameter through, the sample drawing functions are responsible for interpreting the parameter. :param hyperparameter: [dict] hyperparameter space :return: [dict] hyperparameter space """ LOG.debug("convert input parameter\n\n\t{}\n".format(pformat(hyperparameter))) return hyperparameter diff --git a/hyppopy/SolverPool.py b/hyppopy/SolverPool.py index 452dd8d..9eb406f 100644 --- a/hyppopy/SolverPool.py +++ b/hyppopy/SolverPool.py @@ -1,81 +1,81 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, -# Division of Medical and Biological Informatics. +# Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # -# See LICENSE.txt or http://www.mitk.org for details. +# See LICENSE # # Author: Sven Wanner (s.wanner@dkfz.de) from .Singleton import * import os import logging from hyppopy.HyppopyProject import HyppopyProject from hyppopy.solver.OptunaSolver import OptunaSolver from hyppopy.solver.BayesOptSolver import BayesOptSolver from hyppopy.solver.HyperoptSolver import HyperoptSolver from hyppopy.solver.OptunitySolver import OptunitySolver from hyppopy.solver.GridsearchSolver import GridsearchSolver from hyppopy.solver.RandomsearchSolver import RandomsearchSolver from hyppopy.globals import DEBUGLEVEL LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) @singleton_object class SolverPool(metaclass=Singleton): def __init__(self): self._solver_list = ["hyperopt", "optunity", "bayesopt", "optuna", "randomsearch", "gridsearch"] def get_solver_names(self): return self._solver_list def get(self, solver_name=None, project=None): if solver_name is not None: assert isinstance(solver_name, str), "precondition violation, solver_name type str expected, got {} instead!".format(type(solver_name)) if project is not None: assert isinstance(project, HyppopyProject), "precondition violation, project type HyppopyProject expected, got {} instead!".format(type(project)) if "custom_use_solver" in project.__dict__: solver_name = project.custom_use_solver if solver_name not in self._solver_list: raise AssertionError("Solver named [{}] not implemented!".format(solver_name)) if solver_name == "hyperopt": if project is not None: return HyperoptSolver(project) return HyperoptSolver() elif solver_name == "optunity": if project is not None: return OptunitySolver(project) return OptunitySolver() elif solver_name == "bayesopt": if project is not None: return BayesOptSolver(project) return BayesOptSolver() elif solver_name == "optuna": if project is not None: return OptunaSolver(project) return OptunaSolver() elif solver_name == "gridsearch": if project is not None: return GridsearchSolver(project) return GridsearchSolver() elif solver_name == "randomsearch": if project is not None: return RandomsearchSolver(project) return RandomsearchSolver() diff --git a/hyppopy/VirtualFunction.py b/hyppopy/VirtualFunction.py index 044c3c7..67950bc 100644 --- a/hyppopy/VirtualFunction.py +++ b/hyppopy/VirtualFunction.py @@ -1,225 +1,225 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, -# Division of Medical and Biological Informatics. +# Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # -# See LICENSE.txt or http://www.mitk.org for details. +# See LICENSE # # Author: Sven Wanner (s.wanner@dkfz.de) ######################################################################################################################## # USAGE # # The class VirtualFunction is meant to be a virtual energy function with an arbitrary dimensionality. The user can # simply scribble functions as a binary image using e.g. Gimp, defining their ranges using .cfg file and loading them # into the VirtualFunction. An instance of the class can then be used like a normal function returning the sampling of # each dimension loaded. # # 1. create binary images (IMPORTANT same shape for each), background black the function signature white, ensure that # each column has a white pixel. If more than one pixel appears in a column, only the lowest will be used. # # 2. create a .cfg file, see an example in hyppopy/virtualparameterspace # # 3. vfunc = VirtualFunction() # vfunc.load_images(path/of/your/binaryfiles/and/the/configfile) # # 4. use vfunc like a normal function, if you loaded 4 dimension binary images use it like f = vfunc(a,b,c,d) ######################################################################################################################## import os import sys import numpy as np import configparser from glob import glob import matplotlib.pyplot as plt import matplotlib.image as mpimg from hyppopy.globals import VFUNCDATAPATH class VirtualFunction(object): def __init__(self): self.config = None self.data = None self.axis = [] def __call__(self, *args, **kwargs): if len(kwargs) == self.dims(): args = [0]*len(kwargs) for key, value in kwargs.items(): index = int(key.split("_")[1]) args[index] = value assert len(args) == self.dims(), "wrong number of arguments!" for i in range(len(args)): assert self.axis[i][0] <= args[i] <= self.axis[i][1], "out of range access on axis {}!".format(i) lpos, rpos, fracs = self.pos_to_indices(args) fl = self.data[(list(range(self.dims())), lpos)] fr = self.data[(list(range(self.dims())), rpos)] return np.sum(fl*np.array(fracs) + fr*(1-np.array(fracs))) def clear(self): self.axis.clear() self.data = None self.config = None def dims(self): return self.data.shape[0] def size(self): return self.data.shape[1] def range(self, dim): return np.abs(self.axis[dim][1] - self.axis[dim][0]) def minima(self): glob_mins = [] for dim in range(self.dims()): x = [] fmin = np.min(self.data[dim, :]) for _x in range(self.size()): if self.data[dim, _x] <= fmin: x.append(_x/self.size()*(self.axis[dim][1]-self.axis[dim][0])+self.axis[dim][0]) glob_mins.append([x, fmin]) return glob_mins def pos_to_indices(self, positions): lpos = [] rpos = [] pfracs = [] for n in range(self.dims()): pos = positions[n] pos -= self.axis[n][0] pos /= np.abs(self.axis[n][1]-self.axis[n][0]) pos *= self.data.shape[1]-1 lp = int(np.floor(pos)) if lp < 0: lp = 0 rp = int(np.ceil(pos)) if rp > self.data.shape[1]-1: rp = self.data.shape[1]-1 pfracs.append(1.0-(pos-np.floor(pos))) lpos.append(lp) rpos.append(rp) return lpos, rpos, pfracs def plot(self, dim=None, title=""): if dim is None: dim = list(range(self.dims())) else: dim = [dim] fig = plt.figure(figsize=(10, 8)) for i in range(len(dim)): width = np.abs(self.axis[dim[i]][1]-self.axis[dim[i]][0]) ax = np.arange(self.axis[dim[i]][0], self.axis[dim[i]][1], width/self.size()) plt.plot(ax, self.data[dim[i], :], '.', label='axis_{}'.format(str(dim[i]).zfill(2))) plt.legend() plt.grid() plt.title(title) plt.show() def add_dimension(self, data, x_range): if self.data is None: self.data = data if len(self.data.shape) == 1: self.data = self.data.reshape((1, self.data.shape[0])) else: if len(data.shape) == 1: data = data.reshape((1, data.shape[0])) assert self.data.shape[1] == data.shape[1], "shape mismatch while adding dimension!" dims = self.data.shape[0] size = self.data.shape[1] tmp = np.append(self.data, data) self.data = tmp.reshape((dims+1, size)) self.axis.append(x_range) def load_default(self, name="3D"): path = os.path.join(VFUNCDATAPATH, "{}".format(name)) if os.path.exists(path): self.load_images(path) else: raise FileExistsError("No virtualfunction of dimension {} available".format(name)) def load_images(self, path): self.config = None self.data = None self.axis.clear() img_fnames = [] for f in glob(path + os.sep + "*"): if f.endswith(".png"): img_fnames.append(f) elif f.endswith(".cfg"): self.config = self.read_config(f) else: print("WARNING: files of type {} not supported, the file {} is ignored!".format(f.split(".")[-1], os.path.basename(f))) if self.config is None: print("Aborted, failed to read configfile!") sys.exit() sections = self.config.sections() if len(sections) != len(img_fnames): print("Aborted, inconsistent number of image tmplates and axis specifications!") sys.exit() img_fnames.sort() size_x = None size_y = None for n, fname in enumerate(img_fnames): img = mpimg.imread(fname) if len(img.shape) > 2: img = img[:, :, 0] if size_x is None: size_x = img.shape[1] if size_y is None: size_y = img.shape[0] self.data = np.zeros((len(img_fnames), size_x), dtype=np.float32) assert img.shape[0] == size_y, "Shape mismatch in dimension y {} is not {}".format(img.shape[0], size_y) assert img.shape[1] == size_x, "Shape mismatch in dimension x {} is not {}".format(img.shape[1], size_x) self.sample_image(img, n) def sample_image(self, img, dim): sec_name = "axis_{}".format(str(dim).zfill(2)) assert sec_name in self.config.sections(), "config section {} not found!".format(sec_name) settings = self.get_axis_settings(sec_name) self.axis.append([float(settings['min_x']), float(settings['max_x'])]) y_range = [float(settings['min_y']), float(settings['max_y'])] for x in range(img.shape[1]): candidates = np.where(img[:, x] > 0) assert len(candidates[0]) > 0, "non function value in image detected, ensure each column has at least one value > 0!" y_pos = candidates[0][0]/img.shape[0] self.data[dim, x] = 1-y_pos self.data[dim, :] *= np.abs(y_range[1] - y_range[0]) self.data[dim, :] += y_range[0] def read_config(self, fname): try: config = configparser.ConfigParser() config.read(fname) return config except Exception as e: print(e) return None def get_axis_settings(self, section): dict1 = {} options = self.config.options(section) for option in options: try: dict1[option] = self.config.get(section, option) if dict1[option] == -1: print("skip: %s" % option) except: print("exception on %s!" % option) dict1[option] = None return dict1 diff --git a/hyppopy/__init__.py b/hyppopy/__init__.py index d4d866d..6d7ce6a 100644 --- a/hyppopy/__init__.py +++ b/hyppopy/__init__.py @@ -1 +1,16 @@ +# DKFZ +# +# +# Copyright (c) German Cancer Research Center, +# Division of Medical Image Computing. +# All rights reserved. +# +# This software is distributed WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. +# +# See LICENSE +# +# Author: Sven Wanner (s.wanner@dkfz.de) + __version__ = '0.4.0.0' diff --git a/hyppopy/globals.py b/hyppopy/globals.py index 3875fe7..f7347ae 100644 --- a/hyppopy/globals.py +++ b/hyppopy/globals.py @@ -1,34 +1,37 @@ # DKFZ # +# # Copyright (c) German Cancer Research Center, -# Division of Medical and Biological Informatics. +# Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # -# See LICENSE.txt or http://www.mitk.org for details. +# See LICENSE +# +# Author: Sven Wanner (s.wanner@dkfz.de) import os import sys import logging ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) sys.path.insert(0, ROOT) LIBNAME = "hyppopy" TESTDATA_DIR = os.path.join(ROOT, *(LIBNAME, "tests", "data")) HYPERPARAMETERPATH = "hyperparameter" SETTINGSPATH = "settings" VFUNCDATAPATH = os.path.join(os.path.join(ROOT, LIBNAME), "virtualparameterspace") SUPPORTED_DOMAINS = ["uniform", "normal", "loguniform", "categorical"] SUPPORTED_DTYPES = ["int", "float", "str"] DEFAULTITERATIONS = 500 LOGFILENAME = os.path.join(ROOT, '{}_log.log'.format(LIBNAME)) DEBUGLEVEL = logging.DEBUG logging.basicConfig(filename=LOGFILENAME, filemode='w', format='%(levelname)s: %(name)s - %(message)s') diff --git a/hyppopy/tests/test_bayesoptsolver.py b/hyppopy/tests/test_bayesoptsolver.py index 3fb1255..218eeb6 100644 --- a/hyppopy/tests/test_bayesoptsolver.py +++ b/hyppopy/tests/test_bayesoptsolver.py @@ -1,66 +1,66 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, -# Division of Medical and Biological Informatics. +# Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # -# See LICENSE.txt or http://www.mitk.org for details. +# See LICENSE # # Author: Sven Wanner (s.wanner@dkfz.de) import unittest import matplotlib.pylab as plt from hyppopy.solver.BayesOptSolver import * from hyppopy.VirtualFunction import VirtualFunction from hyppopy.HyppopyProject import HyppopyProject class BayesOptSolverTestSuite(unittest.TestCase): def setUp(self): pass def test_solver_complete(self): config = { "hyperparameter": { "axis_00": { "domain": "normal", "data": [300, 800], "type": "float" }, "axis_01": { "domain": "normal", "data": [-1, 1], "type": "float" }, "axis_02": { "domain": "uniform", "data": [0, 10], "type": "float" } }, "settings": { "solver": {"max_iterations": 80}, "custom": {} }} project = HyppopyProject(config) solver = BayesOptSolver(project) vfunc = VirtualFunction() vfunc.load_default() solver.blackbox = vfunc solver.run(print_stats=True) df, best = solver.get_results() self.assertTrue(570 < best['axis_00'] < 590) self.assertTrue(0.1 < best['axis_01'] < 0.8) self.assertTrue(4.5 < best['axis_02'] < 6) if __name__ == '__main__': unittest.main() diff --git a/hyppopy/tests/test_gridsearchsolver.py b/hyppopy/tests/test_gridsearchsolver.py index 6491124..9bea228 100644 --- a/hyppopy/tests/test_gridsearchsolver.py +++ b/hyppopy/tests/test_gridsearchsolver.py @@ -1,206 +1,206 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, -# Division of Medical and Biological Informatics. +# Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # -# See LICENSE.txt or http://www.mitk.org for details. +# See LICENSE # # Author: Sven Wanner (s.wanner@dkfz.de) import unittest from hyppopy.solver.GridsearchSolver import * from hyppopy.VirtualFunction import VirtualFunction from hyppopy.HyppopyProject import HyppopyProject class GridsearchTestSuite(unittest.TestCase): def setUp(self): pass def test_get_uniform_axis_sample(self): drange = [0, 10] N = 11 data = get_uniform_axis_sample(drange[0], drange[1], N, "float") for i in range(11): self.assertEqual(float(i), data[i]) drange = [-10, 10] N = 21 data = get_uniform_axis_sample(drange[0], drange[1], N, "int") self.assertEqual(data[0], -10) self.assertEqual(data[20], 10) self.assertEqual(data[10], 0) def test_get_norm_cdf(self): res = [0, 0.27337265, 0.4331928, 0.48777553, 0.4986501, 0.5013499, 0.51222447, 0.5668072, 0.72662735, 1] f = get_norm_cdf(10) for n, v in enumerate(res): self.assertAlmostEqual(v, f[n]) res = [0.0, 0.27337264762313174, 0.4331927987311419, 0.48777552734495533, 0.4986501019683699, 0.5, 0.5013498980316301, 0.5122244726550447, 0.5668072012688581, 0.7266273523768683, 1.0] f = get_norm_cdf(11) for n, v in enumerate(res): self.assertAlmostEqual(v, f[n]) def test_get_gaussian_axis_sampling(self): res = [-5.0, -2.2662735237686826, -0.6680720126885813, -0.12224472655044671, -0.013498980316301257, 0.013498980316301257, 0.12224472655044671, 0.6680720126885813, 2.2662735237686826, 5.0] bounds = (-5, 5) N = 10 data = get_gaussian_axis_sample(bounds[0], bounds[1], N, "float") for n in range(N): self.assertAlmostEqual(res[n], data[n]) res = [-5.0, -2.2662735237686826, -0.6680720126885813, -0.12224472655044671, -0.013498980316301257, 0.0, 0.013498980316301257, 0.12224472655044671, 0.6680720126885813, 2.2662735237686826, 5.0] bounds = (-5, 5) N = 11 data = get_gaussian_axis_sample(bounds[0], bounds[1], N, "float") for n in range(N): self.assertAlmostEqual(res[n], data[n]) def test_get_logarithmic_axis_sample(self): res = [0.0010000000000000002, 0.0035938136638046297, 0.012915496650148841, 0.046415888336127795, 0.1668100537200059, 0.5994842503189414, 2.154434690031884, 7.7426368268112675, 27.825594022071247, 100.00000000000004] bounds = (0.001, 1e2) N = 10 data = get_logarithmic_axis_sample(bounds[0], bounds[1], N, "float") for n in range(N): self.assertAlmostEqual(res[n], data[n]) res = [0.0010000000000000002, 0.003162277660168382, 0.010000000000000004, 0.03162277660168381, 0.10000000000000006, 0.31622776601683833, 1.0000000000000009, 3.1622776601683813, 10.00000000000001, 31.622776601683846, 100.00000000000004] bounds = (0.001, 1e2) N = 11 data = get_logarithmic_axis_sample(bounds[0], bounds[1], N, "float") for n in range(N): self.assertAlmostEqual(res[n], data[n]) def test_solver(self): config = { "hyperparameter": { "value 1": { "domain": "uniform", "data": [0, 20, 11], "type": "int" }, "value 2": { "domain": "normal", "data": [0, 20.0, 11], "type": "float" }, "value 3": { "domain": "loguniform", "data": [1, 10000, 11], "type": "float" }, "categorical": { "domain": "categorical", "data": ["a", "b"], "type": "str" } }, "settings": { "solver": {}, "custom": {} }} res_labels = ['value 1', 'value 2', 'value 3', 'categorical'] res_values = [[0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20], [0.0, 5.467452952462635, 8.663855974622837, 9.755510546899107, 9.973002039367397, 10.0, 10.026997960632603, 10.244489453100893, 11.336144025377163, 14.532547047537365, 20.0], [1.0, 2.51188643150958, 6.309573444801933, 15.848931924611136, 39.810717055349734, 100.00000000000004, 251.18864315095806, 630.9573444801938, 1584.8931924611143, 3981.071705534977, 10000.00000000001], ['a', 'b'] ] solver = GridsearchSolver(config) searchspace = solver.convert_searchspace(config["hyperparameter"]) for n in range(len(res_labels)): self.assertEqual(res_labels[n], searchspace[0][n]) for i in range(3): self.assertAlmostEqual(res_values[i], searchspace[1][i]) self.assertEqual(res_values[3], searchspace[1][3]) def test_solver_complete(self): config = { "hyperparameter": { "axis_00": { "domain": "normal", "data": [300, 800, 11], "type": "float" }, "axis_01": { "domain": "normal", "data": [-1, 1, 11], "type": "float" }, "axis_02": { "domain": "uniform", "data": [0, 10, 11], "type": "float" } }, "settings": { "solver": {}, "custom": {} }} project = HyppopyProject(config) solver = GridsearchSolver(project) vfunc = VirtualFunction() vfunc.load_default() solver.blackbox = vfunc solver.run(print_stats=False) df, best = solver.get_results() self.assertAlmostEqual(best['axis_00'], 583.40, places=1) self.assertAlmostEqual(best['axis_01'], 0.45, places=1) self.assertAlmostEqual(best['axis_02'], 5.0, places=1) if __name__ == '__main__': unittest.main() diff --git a/hyppopy/tests/test_hyperoptsolver.py b/hyppopy/tests/test_hyperoptsolver.py index 1d2a30e..3713d05 100644 --- a/hyppopy/tests/test_hyperoptsolver.py +++ b/hyppopy/tests/test_hyperoptsolver.py @@ -1,66 +1,66 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, -# Division of Medical and Biological Informatics. +# Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # -# See LICENSE.txt or http://www.mitk.org for details. +# See LICENSE # # Author: Sven Wanner (s.wanner@dkfz.de) import unittest import matplotlib.pylab as plt from hyppopy.solver.HyperoptSolver import * from hyppopy.VirtualFunction import VirtualFunction from hyppopy.HyppopyProject import HyppopyProject class HyperoptSolverTestSuite(unittest.TestCase): def setUp(self): pass def test_solver_complete(self): config = { "hyperparameter": { "axis_00": { "domain": "normal", "data": [300, 800], "type": "float" }, "axis_01": { "domain": "normal", "data": [-1, 1], "type": "float" }, "axis_02": { "domain": "uniform", "data": [0, 10], "type": "float" } }, "settings": { "solver": {"max_iterations": 800}, "custom": {} }} project = HyppopyProject(config) solver = HyperoptSolver(project) vfunc = VirtualFunction() vfunc.load_default() solver.blackbox = vfunc solver.run(print_stats=False) df, best = solver.get_results() self.assertTrue(570 < best['axis_00'] < 590) self.assertTrue(0.1 < best['axis_01'] < 0.8) self.assertTrue(4.5 < best['axis_02'] < 6) if __name__ == '__main__': unittest.main() diff --git a/hyppopy/tests/test_hyppopyproject.py b/hyppopy/tests/test_hyppopyproject.py index 049cf6c..a093ce3 100644 --- a/hyppopy/tests/test_hyppopyproject.py +++ b/hyppopy/tests/test_hyppopyproject.py @@ -1,97 +1,97 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, -# Division of Medical and Biological Informatics. +# Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # -# See LICENSE.txt or http://www.mitk.org for details. +# See LICENSE # # Author: Sven Wanner (s.wanner@dkfz.de) import os import unittest import numpy as np from hyppopy.HyppopyProject import HyppopyProject from hyppopy.globals import TESTDATA_DIR def foo(a, b): return a + b class VirtualFunctionTestSuite(unittest.TestCase): def setUp(self): pass def test_project_creation(self): config = { "hyperparameter": { "C": { "domain": "uniform", "data": [0.0001, 20], "type": "float" }, "kernel": { "domain": "categorical", "data": ["linear", "sigmoid", "poly", "rbf"], "type": "str" } }, "settings": { "solver": { "max_iterations": 300 }, "custom": { "param1": 1, "param2": 2, "function": foo } }} project = HyppopyProject() project.set_config(config) self.assertEqual(project.hyperparameter["C"]["domain"], "uniform") self.assertEqual(project.hyperparameter["C"]["data"], [0.0001, 20]) self.assertEqual(project.hyperparameter["C"]["type"], "float") self.assertEqual(project.hyperparameter["kernel"]["domain"], "categorical") self.assertEqual(project.hyperparameter["kernel"]["data"], ["linear", "sigmoid", "poly", "rbf"]) self.assertEqual(project.hyperparameter["kernel"]["type"], "str") self.assertEqual(project.solver_max_iterations, 300) self.assertEqual(project.custom_param1, 1) self.assertEqual(project.custom_param2, 2) self.assertEqual(project.custom_function(2, 3), 5) self.assertTrue(project.get_typeof("C") is float) self.assertTrue(project.get_typeof("kernel") is str) project.clear() self.assertTrue(len(project.hyperparameter) == 0) self.assertTrue(len(project.settings) == 0) self.assertTrue("solver_max_iterations" not in project.__dict__.keys()) self.assertTrue("custom_param1" not in project.__dict__.keys()) self.assertTrue("custom_param2" not in project.__dict__.keys()) self.assertTrue("custom_function" not in project.__dict__.keys()) project.add_hyperparameter(name="C", domain="uniform", data=[0.0001, 20], dtype="float") project.add_hyperparameter(name="kernel", domain="categorical", data=["linear", "sigmoid", "poly", "rbf"], dtype="str") self.assertEqual(project.hyperparameter["C"]["domain"], "uniform") self.assertEqual(project.hyperparameter["C"]["data"], [0.0001, 20]) self.assertEqual(project.hyperparameter["C"]["type"], "float") self.assertEqual(project.hyperparameter["kernel"]["domain"], "categorical") self.assertEqual(project.hyperparameter["kernel"]["data"], ["linear", "sigmoid", "poly", "rbf"]) self.assertEqual(project.hyperparameter["kernel"]["type"], "str") project.add_settings("solver", "max_iterations", 500) self.assertEqual(project.solver_max_iterations, 500) project.add_settings("solver", "max_iterations", 200) self.assertEqual(project.solver_max_iterations, 200) diff --git a/hyppopy/tests/test_optunasolver.py b/hyppopy/tests/test_optunasolver.py index f22e526..4333544 100644 --- a/hyppopy/tests/test_optunasolver.py +++ b/hyppopy/tests/test_optunasolver.py @@ -1,66 +1,66 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, -# Division of Medical and Biological Informatics. +# Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # -# See LICENSE.txt or http://www.mitk.org for details. +# See LICENSE # # Author: Sven Wanner (s.wanner@dkfz.de) import unittest import matplotlib.pylab as plt from hyppopy.solver.OptunaSolver import * from hyppopy.VirtualFunction import VirtualFunction from hyppopy.HyppopyProject import HyppopyProject class OptunaSolverTestSuite(unittest.TestCase): def setUp(self): pass def test_solver_complete(self): config = { "hyperparameter": { "axis_00": { "domain": "normal", "data": [300, 800], "type": "float" }, "axis_01": { "domain": "normal", "data": [-1, 1], "type": "float" }, "axis_02": { "domain": "uniform", "data": [0, 10], "type": "float" } }, "settings": { "solver": {"max_iterations": 800}, "custom": {} }} project = HyppopyProject(config) solver = OptunaSolver(project) vfunc = VirtualFunction() vfunc.load_default() solver.blackbox = vfunc solver.run(print_stats=False) df, best = solver.get_results() self.assertTrue(570 < best['axis_00'] < 590) self.assertTrue(0.1 < best['axis_01'] < 0.8) self.assertTrue(4.5 < best['axis_02'] < 6) if __name__ == '__main__': unittest.main() diff --git a/hyppopy/tests/test_optunitysolver.py b/hyppopy/tests/test_optunitysolver.py index bba8b36..4d0bc9b 100644 --- a/hyppopy/tests/test_optunitysolver.py +++ b/hyppopy/tests/test_optunitysolver.py @@ -1,66 +1,66 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, -# Division of Medical and Biological Informatics. +# Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # -# See LICENSE.txt or http://www.mitk.org for details. +# See LICENSE # # Author: Sven Wanner (s.wanner@dkfz.de) import unittest import matplotlib.pylab as plt from hyppopy.solver.OptunitySolver import * from hyppopy.VirtualFunction import VirtualFunction from hyppopy.HyppopyProject import HyppopyProject class OptunitySolverTestSuite(unittest.TestCase): def setUp(self): pass def test_solver_complete(self): config = { "hyperparameter": { "axis_00": { "domain": "normal", "data": [300, 800], "type": "float" }, "axis_01": { "domain": "normal", "data": [-1, 1], "type": "float" }, "axis_02": { "domain": "uniform", "data": [0, 10], "type": "float" } }, "settings": { "solver": {"max_iterations": 800}, "custom": {} }} project = HyppopyProject(config) solver = OptunitySolver(project) vfunc = VirtualFunction() vfunc.load_default() solver.blackbox = vfunc solver.run(print_stats=False) df, best = solver.get_results() self.assertTrue(570 < best['axis_00'] < 590) self.assertTrue(0.1 < best['axis_01'] < 0.8) self.assertTrue(4.5 < best['axis_02'] < 6) if __name__ == '__main__': unittest.main() diff --git a/hyppopy/tests/test_randomsearchsolver.py b/hyppopy/tests/test_randomsearchsolver.py index 946e352..6e18dd0 100644 --- a/hyppopy/tests/test_randomsearchsolver.py +++ b/hyppopy/tests/test_randomsearchsolver.py @@ -1,132 +1,132 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, -# Division of Medical and Biological Informatics. +# Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # -# See LICENSE.txt or http://www.mitk.org for details. +# See LICENSE # # Author: Sven Wanner (s.wanner@dkfz.de) import unittest import matplotlib.pylab as plt from hyppopy.solver.RandomsearchSolver import * from hyppopy.VirtualFunction import VirtualFunction from hyppopy.HyppopyProject import HyppopyProject class RandomsearchTestSuite(unittest.TestCase): def setUp(self): pass def test_draw_uniform_sample(self): param = {"data": [0, 1, 10], "type": "float"} values = [] for i in range(10000): values.append(draw_uniform_sample(param)) self.assertTrue(0 <= values[-1] <= 1) self.assertTrue(isinstance(values[-1], float)) hist = plt.hist(values, bins=10, normed=True) std = np.std(hist[0]) mean = np.mean(hist[0]) self.assertTrue(std < 0.05) self.assertTrue(0.9 < mean < 1.1) param = {"data": [0, 10, 11], "type": "int"} values = [] for i in range(10000): values.append(draw_uniform_sample(param)) self.assertTrue(0 <= values[-1] <= 10) self.assertTrue(isinstance(values[-1], int)) hist = plt.hist(values, bins=11, normed=True) std = np.std(hist[0]) mean = np.mean(hist[0]) self.assertTrue(std < 0.05) self.assertTrue(0.09 < mean < 0.11) def test_draw_normal_sample(self): param = {"data": [0, 10, 11], "type": "int"} values = [] for i in range(10000): values.append(draw_normal_sample(param)) self.assertTrue(0 <= values[-1] <= 10) self.assertTrue(isinstance(values[-1], int)) hist = plt.hist(values, bins=11, normed=True) for i in range(1, 5): self.assertTrue(hist[0][i-1]-hist[0][i] < 0) for i in range(5, 10): self.assertTrue(hist[0][i] - hist[0][i+1] > 0) def test_draw_loguniform_sample(self): param = {"data": [1, 1000, 11], "type": "float"} values = [] for i in range(10000): values.append(draw_loguniform_sample(param)) self.assertTrue(1 <= values[-1] <= 1000) self.assertTrue(isinstance(values[-1], float)) hist = plt.hist(values, bins=11, normed=True) for i in range(4): self.assertTrue(hist[0][i] > hist[0][i+1]) self.assertTrue((hist[0][i] - hist[0][i+1]) > 0) def test_draw_categorical_sample(self): param = {"data": [1, 2, 3], "type": int} values = [] for i in range(10000): values.append(draw_categorical_sample(param)) self.assertTrue(values[-1] == 1 or values[-1] == 2 or values[-1] == 3) self.assertTrue(isinstance(values[-1], int)) hist = plt.hist(values, bins=3, normed=True) for i in range(3): self.assertTrue(0.45 < hist[0][i] < 0.55) def test_solver_complete(self): config = { "hyperparameter": { "axis_00": { "domain": "normal", "data": [300, 800], "type": "float" }, "axis_01": { "domain": "uniform", "data": [-1, 1], "type": "float" }, "axis_02": { "domain": "uniform", "data": [0, 10], "type": "float" } }, "settings": { "solver": {"max_iterations": 8000}, "custom": {} }} project = HyppopyProject(config) solver = RandomsearchSolver(project) vfunc = VirtualFunction() vfunc.load_default() solver.blackbox = vfunc solver.run(print_stats=False) df, best = solver.get_results() self.assertTrue(570 < best['axis_00'] < 590) self.assertTrue(0.1 < best['axis_01'] < 0.8) self.assertTrue(4.5 < best['axis_02'] < 6) if __name__ == '__main__': unittest.main() diff --git a/hyppopy/tests/test_virtualfunction.py b/hyppopy/tests/test_virtualfunction.py index 83c3b6c..f807da0 100644 --- a/hyppopy/tests/test_virtualfunction.py +++ b/hyppopy/tests/test_virtualfunction.py @@ -1,96 +1,96 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, -# Division of Medical and Biological Informatics. +# Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # -# See LICENSE.txt or http://www.mitk.org for details. +# See LICENSE # # Author: Sven Wanner (s.wanner@dkfz.de) import os import unittest import numpy as np from hyppopy.VirtualFunction import VirtualFunction from hyppopy.globals import TESTDATA_DIR class VirtualFunctionTestSuite(unittest.TestCase): def setUp(self): pass def test_imagereading(self): vfunc = VirtualFunction() vfunc.load_images(os.path.join(TESTDATA_DIR, 'functionsimulator')) self.assertTrue(isinstance(vfunc.data, np.ndarray)) self.assertEqual(vfunc.data.shape[0], 5) self.assertEqual(vfunc.data.shape[1], 512) gt = [0.83984375*5, 0.44140625*20-10, 0.25390625*20, 0.81640625*8-10, 0.67578125*2+2] for i in range(5): self.assertAlmostEqual(vfunc.data[i][0], gt[i]) gt = [[0, 1], [-10, 10], [0, 20], [-30, 5], [5, 10]] for i in range(5): self.assertEqual(vfunc.axis[i][0], gt[i][0]) self.assertEqual(vfunc.axis[i][1], gt[i][1]) def test_data_adding(self): gt = [[-10, 10], [-30, 5]] vfunc = VirtualFunction() dim0 = np.arange(0, 1.1, 0.1) dim1 = np.arange(1.0, -0.1, -0.1) vfunc.add_dimension(dim0, gt[0]) self.assertEqual(len(vfunc.data.shape), 2) self.assertEqual(vfunc.data.shape[0], 1) self.assertEqual(vfunc.data.shape[1], 11) vfunc.add_dimension(dim1, gt[1]) self.assertEqual(vfunc.data.shape[0], 2) self.assertEqual(vfunc.data.shape[1], 11) for n in range(11): self.assertAlmostEqual(dim0[n], vfunc.data[0, n]) self.assertAlmostEqual(dim1[n], vfunc.data[1, n]) for i in range(2): self.assertEqual(vfunc.axis[i][0], gt[i][0]) self.assertEqual(vfunc.axis[i][1], gt[i][1]) def test_minima(self): vfunc = VirtualFunction() vfunc.load_images(os.path.join(TESTDATA_DIR, 'functionsimulator')) minima = vfunc.minima() gt = [[[0.7265625], 0.48828125], [[-4.0234375], -7.890625], [[2.265625], 0.859375], [ [-17.421875, -17.353515625, -17.28515625, -17.216796875, -17.1484375, -17.080078125, -17.01171875, -16.943359375, -16.875, -16.806640625, -16.73828125, -16.669921875, -16.6015625, -16.533203125, -16.46484375, -16.396484375, -16.328125, -16.259765625, -16.19140625, -16.123046875, -16.0546875, -15.986328125, -15.91796875, -15.849609375, -15.78125, -15.712890625, -15.64453125, -15.576171875, -15.5078125, -15.439453125, -15.37109375, -15.302734375, -15.234375, -15.166015625, -15.09765625, -15.029296875, -14.9609375, -14.892578125, -14.82421875, -14.755859375, -14.6875, -14.619140625, -14.55078125, -14.482421875, -14.4140625, -14.345703125, -14.27734375, -14.208984375, -14.140625, -14.072265625, -14.00390625, -13.935546875, -13.8671875, -13.798828125, -13.73046875, -13.662109375, -13.59375, -13.525390625, -13.45703125, -13.388671875, -13.3203125, -13.251953125, -13.18359375, -13.115234375, -13.046875, -12.978515625, -12.91015625, -12.841796875, -12.7734375, -12.705078125, -12.63671875, -12.568359375, -12.5, -12.431640625, -12.36328125, -12.294921875, -12.2265625, -12.158203125, -12.08984375, -12.021484375, -11.953125, -11.884765625, -11.81640625, -11.748046875, -11.6796875, -11.611328125, -11.54296875, -11.474609375, -11.40625, -11.337890625, -11.26953125, -11.201171875, -11.1328125, -11.064453125, -10.99609375, -10.927734375, -10.859375, -10.791015625, -10.72265625, -10.654296875, -10.5859375, -10.517578125, -10.44921875, -10.380859375, -10.3125, -10.244140625, -10.17578125, -10.107421875, -10.0390625, -9.970703125, -9.90234375, -9.833984375, -9.765625, -9.697265625, -9.62890625, -9.560546875, -9.4921875, -9.423828125, -9.35546875, -9.287109375, -9.21875, -9.150390625, -9.08203125, -9.013671875, -8.9453125, -8.876953125, -8.80859375, -8.740234375, -8.671875, -8.603515625, -8.53515625, -8.466796875, -8.3984375, -8.330078125, -8.26171875, -8.193359375, -8.125, -8.056640625, -7.98828125, -7.919921875, -7.8515625, -7.783203125, -7.71484375, -7.646484375, -7.578125, -7.509765625, -7.44140625, -7.373046875, -7.3046875, -7.236328125, -7.16796875, -7.099609375, -7.03125], -9.125], [[5.44921875, 5.458984375, 5.46875, 5.478515625, 5.48828125, 5.498046875, 5.5078125, 5.517578125, 5.52734375], 2.09375]] self.assertAlmostEqual(minima, gt) if __name__ == '__main__': unittest.main() diff --git a/setup.py b/setup.py index c88d5db..9975dc0 100644 --- a/setup.py +++ b/setup.py @@ -1,55 +1,54 @@ import os from setuptools import setup, find_packages with open('README.md') as f: readme = f.read() with open('LICENSE') as f: license = f.read() VERSION = "0.4.0.1" ROOT = os.path.dirname(os.path.realpath(__file__)) new_init = [] with open(os.path.join(ROOT, *("hyppopy", "__init__.py")), "r") as infile: for line in infile: new_init.append(line) for n in range(len(new_init)): if new_init[n].startswith("__version__"): split = line.split("=") new_init[n] = "__version__ = '" + VERSION + "'\n" with open(os.path.join(ROOT, *("hyppopy", "__init__.py")), "w") as outfile: outfile.writelines(new_init) setup( name='hyppopy', version=VERSION, description='Hyper-Parameter Optimization Toolbox for Blackboxfunction Optimization', long_description=readme, # if you want, put your own name here # (this would likely result in people sending you emails) author='Sven Wanner', author_email='s.wanner@dkfz.de', url='', license=license, packages=find_packages(exclude=('tests', 'doc')), - package_data={ - }, + package_data={}, # the requirements to install this project. # Since this one is so simple this is empty. install_requires=[ - 'bayesian-optimization>=1.0.1', - 'hyperopt>=0.1.2', - 'matplotlib>=3.0.3', - 'numpy>=1.16.2', - 'optuna>=0.9.0', - 'Optunity>=1.1.1', - 'pandas>=0.24.2', - 'pytest>=4.3.1', - 'scikit-learn>=0.20.3', - 'scipy>=1.2.1' + 'bayesian-optimization>=1.0.1', + 'hyperopt>=0.1.2', + 'matplotlib>=3.0.3', + 'numpy>=1.16.2', + 'optuna>=0.9.0', + 'Optunity>=1.1.1', + 'pandas>=0.24.2', + 'pytest>=4.3.1', + 'scikit-learn>=0.20.3', + 'scipy>=1.2.1' ], )