diff --git a/LICENSE b/LICENSE index 366acc1..1422535 100644 --- a/LICENSE +++ b/LICENSE @@ -1,38 +1,38 @@ ======================================================================= -Copyright (c) 2003-2012 German Cancer Research Center, +Copyright (c) 2019 German Cancer Research Center, Division of Medical Image Computing All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the German Cancer Research Center, nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ======================================================================= diff --git a/README.md b/README.md index 17aa081..b1339d8 100644 --- a/README.md +++ b/README.md @@ -1,353 +1,374 @@                 ![docs_title_logo](./resources/docs_title_logo.png) # A Hyper-Parameter Optimization Toolbox
## What is Hyppopy? Hyppopy is a python toolbox for blackbox optimization. It's purpose is to offer a unified and easy to use interface to a collection of solver libraries. Currently provided solvers are: * [Hyperopt](http://hyperopt.github.io/hyperopt/) * [Optunity](https://optunity.readthedocs.io/en/latest/user/index.html) * [Optuna](https://optuna.org/) * [BayesianOptimization](https://github.com/fmfn/BayesianOptimization) * Randomsearch Solver * Gridsearch Solver ## Installation 1. clone the [Hyppopy](http:\\github.com) project from Github 2. (create a virtual environment), open a console (with your activated virtual env) and go to the hyppopy root folder 3. ```$ pip install -r requirements.txt``` 4. ```$ python setup.py install``` ## How to use Hyppopy? #### The HyppopyProject class The HyppopyProject class takes care all settings necessary for the solver and your workflow. To setup a HyppopyProject instance we can use a nested dictionary or the classes memberfunctions respectively. ```python # Import the HyppopyProject class from hyppopy.HyppopyProject import HyppopyProject # Create a nested dict with a section hyperparameter. We define a 2 dimensional # hyperparameter space with a numerical dimension named myNumber of type float and # a uniform sampling. The second dimension is a categorical parameter of type string. config = { "hyperparameter": { "myNumber": { "domain": "uniform", "data": [0, 100], "type": "float" }, "myOption": { "domain": "categorical", "data": ["a", "b", "c"], "type": "str" } }} # Create a HyppopyProject instance and pass the config dict to # the constructor. Alternatively one can use set_config method. project = HyppopyProject(config=config) # To demonstrate the second option we clear the project project.clear() # and add the parameter again using the member function add_hyperparameter project.add_hyperparameter(name="myNumber", domain="uniform", data=[0, 100], dtype="float") project.add_hyperparameter(name="myOption", domain="categorical", data=["a", "b", "c"], dtype="str") ``` ```python from hyppopy.HyppopyProject import HyppopyProject # We might have seen a warning: 'UserWarning: config dict had no # section settings/solver/max_iterations, set default value: 500' # when executing the example above. This is due to the fact that # most solvers need a value for a maximum number of iterations. # To take care of solver settings (there might be more in the future) # one can set a second section called settings. The settings section # again is splitted into a subsection 'solver' and a subsection 'custom'. # When adding max_iterations to the section settings/solver we can change # the number of iterations the solver is doing. All solver except of the # GridsearchSolver make use of the value max_iterations. # The usage of the custom section is demonstrated later. config = { "hyperparameter": { "myNumber": { "domain": "uniform", "data": [0, 100], "type": "float" }, "myOption": { "domain": "categorical", "data": ["a", "b", "c"], "type": "str" } }, "settings": { "solver": { "max_iterations": 500 }, "custom": {} }} project = HyppopyProject(config=config) ``` The settings added are automatically converted to a class member with a prefix_ where prefix is the name of the subsection. One can make use of this feature to build custom workflows by adding params to the custom section. More interesting is this feature when developing your own solver. ```python from hyppopy.HyppopyProject import HyppopyProject # Creating a HyppopyProject instance project = HyppopyProject() project.add_hyperparameter(name="x", domain="uniform", data=[-10, 10], dtype="float") project.add_hyperparameter(name="y", domain="uniform", data=[-10, 10], dtype="float") project.add_settings(section="solver", name="max_iterations", value=300) project.add_settings(section="custom", name="my_param1", value=True) project.add_settings(section="custom", name="my_param2", value=42) print("What is max_iterations value? {}".format(project.solver_max_iterations)) if project.custom_my_param1: print("What is the answer? {}".format(project.custom_my_param2)) else: print("What is the answer? x") ``` #### The HyppopySolver classes Each solver is a child of the HyppopySolver class. This is only interesting if you're planning to write a new solver, we will discuss this in the section Solver Development. All solvers we can use to optimize our blackbox function are part of the module 'hyppopy.solver'. Below is a list of all solvers available along with their access key in squared brackets. * HyperoptSolver [hyperopt] * OptunitySolver [optunity] * OptunaSolver [optuna] * BayesOptSolver [bayesopt] * RandomsearchSolver [randomsearch] * GridsearchSolver [gridsearch] There are two options to get a solver, we can import directly from the hyppopy.solver package or we use the SolverPool class. We look into both options by optimizing a simple function, starting with the direct import case. ```python # Import the HyppopyProject class from hyppopy.HyppopyProject import HyppopyProject # Import the HyperoptSolver class, in this case wh use Hyperopt from hyppopy.solver.HyperoptSolver import HyperoptSolver # Our function to optimize def my_loss_func(x, y): return x**2+y**2 # Creating a HyppopyProject instance project = HyppopyProject() project.add_hyperparameter(name="x", domain="uniform", data=[-10, 10], dtype="float") project.add_hyperparameter(name="y", domain="uniform", data=[-10, 10], dtype="float") project.add_settings(section="solver", name="max_iterations", value=300) # create a solver instance solver = HyperoptSolver(project) # pass the loss function to the solver solver.blackbox = my_loss_func # run the solver solver.run() df, best = solver.get_results() print("\n") print("*"*100) print("Best Parameter Set:\n{}".format(best)) print("*"*100) ``` The SolverPool is a class keeping track of all solver classes. We have several options to ask the SolverPool for the desired solver. We can add an option called use_solver to our settings/custom section or to the project instance respectively, or we can use the solver access key (see solver listing above) to ask for the solver directly. ```python # import the SolverPool class from hyppopy.SolverPool import SolverPool # Import the HyppopyProject class from hyppopy.HyppopyProject import HyppopyProject # Our function to optimize def my_loss_func(x, y): return x**2+y**2 # Creating a HyppopyProject instance project = HyppopyProject() project.add_hyperparameter(name="x", domain="uniform", data=[-10, 10], dtype="float") project.add_hyperparameter(name="y", domain="uniform", data=[-10, 10], dtype="float") project.add_settings(section="solver", name="max_iterations", value=300) project.add_settings(section="custom", name="use_solver", value="hyperopt") # create a solver instance. The SolverPool class is a singleton # and can be used without instanciating. It looks in the project # instance for the use_solver option and returns the correct solver. solver = SolverPool.get(project=project) # Another option without the usage of the use_solver field would be: # solver = SolverPool.get(solver_name='hyperopt', project=project) # pass the loss function to the solver solver.blackbox = my_loss_func # run the solver solver.run() df, best = solver.get_results() print("\n") print("*"*100) print("Best Parameter Set:\n{}".format(best)) print("*"*100) ``` #### The BlackboxFunction class To extend the possibilities beyond using parameter only loss function as in the examples above, the BlackboxFunction class can be used. This class is a wrapper class around the actual loss_function providing a more advanced access to data handling and a callback_function for accessing the solvers iteration loop. ```python # import the HyppopyProject class keeping track of inputs from hyppopy.HyppopyProject import HyppopyProject # import the SolverPool singleton class from hyppopy.SolverPool import SolverPool # import the Blackboxfunction class wrapping your problem for Hyppopy from hyppopy.BlackboxFunction import BlackboxFunction # Create the HyppopyProject class instance project = HyppopyProject() project.add_hyperparameter(name="C", domain="uniform", data=[0.0001, 20], dtype="float") project.add_hyperparameter(name="gamma", domain="uniform", data=[0.0001, 20], dtype="float") project.add_hyperparameter(name="kernel", domain="categorical", data=["linear", "sigmoid", "poly", "rbf"], dtype="str") project.add_settings(section="solver", name="max_iterations", value=500) project.add_settings(section="custom", name="use_solver", value="optunity") # The BlackboxFunction signature is as follows: # BlackboxFunction(blackbox_func=None, # dataloader_func=None, # preprocess_func=None, # callback_func=None, # data=None, # **kwargs) # # - blackbox_func: a function pointer to the users loss function # - dataloader_func: a function pointer for handling dataloading. The function is called once before # optimizing. What it returns is passed as first argument to your loss functions # data argument. # - preprocess_func: a function pointer for data preprocessing. The function is called once before # optimizing and gets via kwargs['data'] the raw data object set directly or returned # from dataloader_func. What this function returns is then what is passed as first # argument to your loss function. # - callback_func: a function pointer called after each iteration. The input kwargs is a dictionary # keeping the parameters used in this iteration, the 'iteration' index, the 'loss' # and the 'status'. The function in this example is used for realtime printing it's # input but can also be used for realtime visualization. # - data: if not done via dataloader_func one can set a raw_data object directly # - kwargs: dict that whose content is passed to all functions above. from sklearn.svm import SVC from sklearn.datasets import load_iris from sklearn.model_selection import cross_val_score def my_dataloader_function(**kwargs): print("Dataloading...") # kwargs['params'] allows accessing additional parameter passed, # see below my_preproc_param, my_dataloader_input. print("my loading argument: {}".format(kwargs['params']['my_dataloader_input'])) iris_data = load_iris() return [iris_data.data, iris_data.target] def my_preprocess_function(**kwargs): print("Preprocessing...") # kwargs['data'] allows accessing the input data print("data:", kwargs['data'][0].shape, kwargs['data'][1].shape) # kwargs['params'] allows accessing additional parameter passed, # see below my_preproc_param, my_dataloader_input. print("kwargs['params']['my_preproc_param']={}".format(kwargs['params']['my_preproc_param']), "\n") # if the preprocessing function returns something, # the input data will be replaced with the data returned by this function. x = kwargs['data'][0] y = kwargs['data'][1] for i in range(x.shape[0]): x[i, :] += kwargs['params']['my_preproc_param'] return [x, y] def my_callback_function(**kwargs): print("\r{}".format(kwargs), end="") def my_loss_function(data, params): clf = SVC(**params) return -cross_val_score(estimator=clf, X=data[0], y=data[1], cv=3).mean() # We now create the BlackboxFunction object and pass all function pointers defined above, # as well as 2 dummy parameter (my_preproc_param, my_dataloader_input) for demonstration purposes. blackbox = BlackboxFunction(blackbox_func=my_loss_function, dataloader_func=my_dataloader_function, preprocess_func=my_preprocess_function, callback_func=my_callback_function, my_preproc_param=1, my_dataloader_input='could/be/a/path') # Get the solver solver = SolverPool.get(project=project) # Give the solver your blackbox solver.blackbox = blackbox # Run the solver solver.run() # Get your results df, best = solver.get_results() print("\n") print("*"*100) print("Best Parameter Set:\n{}".format(best)) print("*"*100) ``` #### The Parameter Space Domains Each hyperparameter needs a range and a domain specifier. The range, specified via 'data', is the left and right bound of an interval (!!!exception is the domain 'categorical', here 'data' is the actual list of data elements!!!) and the domain specifier the way this interval is sampled. Currently supported domains are: * uniform (samples the interval [a,b] evenly) * normal (a gaussian sampling of the interval [a,b] such that mu=a+(b-a)/2 and sigma=(b-a)/6) * loguniform (a logaritmic sampling of the iterval [a,b], such that the exponent e^x is sampled evenly x=[log(a),log(b)]) * categorical (in this case data is not interpreted as interval but as actual list of objects) One exception is the GridsearchSolver, here we need to specifiy an interval and a number of samples like so: 'data': [a,b,N]. The max_iterations parameter is obsolet in this case because each axis specifies an individual number of samples. ```python # import the SolverPool class from hyppopy.solver.GridsearchSolver import GridsearchSolver # Import the HyppopyProject class from hyppopy.HyppopyProject import HyppopyProject # Our function to optimize def my_loss_func(x, y): return x**2+y**2 # Creating a HyppopyProject instance project = HyppopyProject() project.add_hyperparameter(name="x", domain="uniform", data=[-1.1, 1, 10], dtype="float") project.add_hyperparameter(name="y", domain="uniform", data=[-1.1, 1, 12], dtype="float") solver = GridsearchSolver(project=project) # pass the loss function to the solver solver.blackbox = my_loss_func # run the solver solver.run() df, best = solver.get_results() print("\n") print("*"*100) print("Best Parameter Set:\n{}".format(best)) print("*"*100) ``` + +#### Using a Visdom Server to Visualize the Optimization Process + +We can simply create a realtime visualization using a visdom server. If installed, start your visdom server via console command: +``` +>visdom +``` + +Go to your browser and open the site: http://localhost:8097 + +To enable the visualization call the function 'start_viewer' before running the solver: + +```python +#enable visualization +solver.start_viewer() +# Run the solver +solver.run() +``` + +You can also change the port and the server name in start_viewer(port=8097, server="http://localhost") + diff --git a/examples/solver_comparison.py b/examples/solver_comparison.py index 9dd2322..66247d1 100644 --- a/examples/solver_comparison.py +++ b/examples/solver_comparison.py @@ -1,196 +1,194 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE -# -# Author: Sven Wanner (s.wanner@dkfz.de) import os import pickle import numpy as np from math import pi from pprint import pprint import matplotlib.pyplot as plt from hyppopy.SolverPool import SolverPool from hyppopy.HyppopyProject import HyppopyProject from hyppopy.VirtualFunction import VirtualFunction from hyppopy.BlackboxFunction import BlackboxFunction OUTPUTDIR = "C:\\Users\\s635r\\Desktop\\solver_comparison" SOLVER = ["hyperopt", "optunity", "randomsearch", "optuna"]#, "bayesopt"] ITERATIONS = [25, 100, 250, 500] STATREPEATS = 10 VFUNC = "5D3" OVERWRITE = False OUTPUTDIR = os.path.join(OUTPUTDIR, VFUNC) if not os.path.isdir(OUTPUTDIR): os.makedirs(OUTPUTDIR) def compute_deviation(solver_name, vfunc_id, iterations, N, fname): project = HyppopyProject() project.add_hyperparameter(name="axis_00", domain="uniform", data=[0, 1], dtype="float") project.add_hyperparameter(name="axis_01", domain="uniform", data=[0, 1], dtype="float") project.add_hyperparameter(name="axis_02", domain="uniform", data=[0, 1], dtype="float") project.add_hyperparameter(name="axis_03", domain="uniform", data=[0, 1], dtype="float") project.add_hyperparameter(name="axis_04", domain="uniform", data=[0, 1], dtype="float") vfunc = VirtualFunction() vfunc.load_default(vfunc_id) minima = vfunc.minima() def my_loss_function(data, params): return vfunc(**params) blackbox = BlackboxFunction(data=[], blackbox_func=my_loss_function) results = {} results["gt"] = [] for mini in minima: results["gt"].append(np.median(mini[0])) for iter in iterations: results[iter] = {"minima": {}, "loss": None} for i in range(vfunc.dims()): results[iter]["minima"]["axis_0{}".format(i)] = [] project.add_settings(section="solver", name="max_iterations", value=iter) project.add_settings(section="custom", name="use_solver", value=solver_name) solver = SolverPool.get(project=project) solver.blackbox = blackbox axis_minima = [] best_losses = [] for i in range(vfunc.dims()): axis_minima.append([]) for n in range(N): print("\rSolver={} iteration={} round={}".format(solver, iter, n), end="") solver.run(print_stats=False) df, best = solver.get_results() best_row = df['losses'].idxmin() best_losses.append(df['losses'][best_row]) for i in range(vfunc.dims()): tmp = df['axis_0{}'.format(i)][best_row] axis_minima[i].append(tmp) for i in range(vfunc.dims()): results[iter]["minima"]["axis_0{}".format(i)] = [np.mean(axis_minima[i]), np.std(axis_minima[i])] results[iter]["loss"] = [np.mean(best_losses), np.std(best_losses)] file = open(fname, 'wb') pickle.dump(results, file) file.close() def make_radarplot(results, title, fname=None): gt = results.pop("gt") categories = list(results[list(results.keys())[0]]["minima"].keys()) N = len(categories) angles = [n / float(N) * 2 * pi for n in range(N)] angles += angles[:1] ax = plt.subplot(1, 1, 1, polar=True, ) ax.set_theta_offset(pi / 2) ax.set_theta_direction(-1) plt.xticks(angles[:-1], categories, color='grey', size=8) ax.set_rlabel_position(0) plt.yticks([0.2, 0.4, 0.6, 0.8, 1.0], ["0.2", "0.4", "0.6", "0.8", "1.0"], color="grey", size=7) plt.ylim(0, 1) gt += gt[:1] ax.fill(angles, gt, color=(0.2, 0.8, 0.2), alpha=0.2) colors = [] cm = plt.get_cmap('Set1') if len(results) > 2: indices = list(range(0, len(results) + 1)) indices.pop(2) else: indices = list(range(0, len(results))) for i in range(len(results)): colors.append(cm(indices[i])) for iter, data in results.items(): values = [] for i in range(len(categories)): values.append(data["minima"]["axis_0{}".format(i)][0]) values += values[:1] color = colors.pop(0) ax.plot(angles, values, color=color, linewidth=2, linestyle='solid', label="iterations {}".format(iter)) plt.title(title, size=11, color=(0.1, 0.1, 0.1), y=1.1) plt.legend(bbox_to_anchor=(0.08, 1.12)) if fname is None: plt.show() else: plt.savefig(fname + ".png") plt.savefig(fname + ".svg") plt.clf() def make_deviationerrorplot(fnames): results = {} for fname in fnames: file = open(fname, 'rb') result = pickle.load(file) file.close() results[os.path.basename(fname)] = result pprint(results) plt.figure() for iter in results["hyperopt"].keys(): y = [] if iter == "gt": x = list(range(len(results["hyperopt"][iter]))) for i in range(len(results["hyperopt"][iter])): y.append(results["hyperopt"][iter][i]) plt.plot(x, y, "--g", label="groundtruth: {}".format(iter)) continue x = list(range(len(results["hyperopt"][iter]["minima"]))) for i in range(len(results["hyperopt"][iter]["minima"])): y.append(results["hyperopt"][iter]["minima"]["axis_0{}".format(i)][0]) plt.plot(x, y, label="iterations: {}".format(iter)) plt.title("") plt.legend() plt.show() ################################################## ############### create datasets ################## fnames = [] for solver_name in SOLVER: fname = os.path.join(OUTPUTDIR, solver_name) fnames.append(fname) if OVERWRITE or not os.path.isfile(fname): compute_deviation(solver_name, VFUNC, ITERATIONS, N=STATREPEATS, fname=fname) ################################################## ################################################## ################################################## ############## create radarplots ################# for solver_name, fname in zip(SOLVER, fnames): file = open(fname, 'rb') results = pickle.load(file) file.close() make_radarplot(results, solver_name, fname + "_deviation") ################################################## ################################################## diff --git a/examples/tutorial_custom_visualization.py b/examples/tutorial_custom_visualization.py index cf463c1..ab3b706 100644 --- a/examples/tutorial_custom_visualization.py +++ b/examples/tutorial_custom_visualization.py @@ -1,107 +1,105 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE -# -# Author: Sven Wanner (s.wanner@dkfz.de) import matplotlib.pylab as plt from hyppopy.SolverPool import SolverPool from hyppopy.HyppopyProject import HyppopyProject from hyppopy.VirtualFunction import VirtualFunction from hyppopy.BlackboxFunction import BlackboxFunction project = HyppopyProject() project.add_hyperparameter(name="axis_00", domain="uniform", data=[0, 1], dtype="float") project.add_hyperparameter(name="axis_01", domain="uniform", data=[0, 800], dtype="float") project.add_hyperparameter(name="axis_02", domain="uniform", data=[0, 5], dtype="float") project.add_hyperparameter(name="axis_03", domain="uniform", data=[1, 10000], dtype="float") project.add_hyperparameter(name="axis_04", domain="uniform", data=[0, 10], dtype="float") project.add_settings(section="solver", name="max_iterations", value=500) project.add_settings(section="custom", name="use_solver", value="randomsearch") plt.ion() fig, axes = plt.subplots(nrows=2, ncols=3, figsize=(12, 8), sharey=True) plot_data = {"iterations": [], "loss": [], "axis_00": [], "axis_01": [], "axis_02": [], "axis_03": [], "axis_04": []} def my_visualization_function(**kwargs): print("\r{}".format(kwargs), end="") plot_data["iterations"].append(kwargs['iterations']) plot_data["loss"].append(kwargs['loss']) plot_data["axis_00"].append(kwargs['axis_00']) plot_data["axis_01"].append(kwargs['axis_01']) plot_data["axis_02"].append(kwargs['axis_02']) plot_data["axis_03"].append(kwargs['axis_03']) plot_data["axis_04"].append(kwargs['axis_04']) axes[0, 0].clear() axes[0, 0].scatter(plot_data["axis_00"], plot_data["loss"], c=plot_data["loss"], cmap="jet", marker='.') axes[0, 0].set_ylabel("loss") axes[0, 0].set_xlabel("axis_00") axes[0, 1].clear() axes[0, 1].scatter(plot_data["axis_01"], plot_data["loss"], c=plot_data["loss"], cmap="jet", marker='.') axes[0, 1].set_xlabel("axis_01") axes[0, 2].clear() axes[0, 2].scatter(plot_data["axis_02"], plot_data["loss"], c=plot_data["loss"], cmap="jet", marker='.') axes[0, 2].set_xlabel("axis_02") axes[1, 0].clear() axes[1, 0].scatter(plot_data["axis_03"], plot_data["loss"], c=plot_data["loss"], cmap="jet", marker='.') axes[1, 0].set_ylabel("loss") axes[1, 0].set_xlabel("axis_03") axes[1, 1].clear() axes[1, 1].scatter(plot_data["axis_04"], plot_data["loss"], c=plot_data["loss"], cmap="jet", marker='.') axes[1, 1].set_xlabel("axis_04") axes[1, 2].clear() axes[1, 2].plot(plot_data["iterations"], plot_data["loss"], "--", c=(0.8, 0.8, 0.8, 0.5)) axes[1, 2].scatter(plot_data["iterations"], plot_data["loss"], marker='.', c=(0.2, 0.2, 0.2)) axes[1, 2].set_xlabel("iterations") plt.draw() plt.tight_layout() plt.pause(0.001) def my_loss_function(data, params): vfunc = VirtualFunction() vfunc.load_default(5) return vfunc(**params) blackbox = BlackboxFunction(data=[], blackbox_func=my_loss_function, callback_func=my_visualization_function) solver = SolverPool.get(project=project) solver.blackbox = blackbox solver.run() df, best = solver.get_results() print("\n") print("*" * 100) print("Best Parameter Set:\n{}".format(best)) print("*" * 100) print("") save_plot = input("Save Plot? [y/n] ") if save_plot == "y": plt.savefig('plot_{}.png'.format(project.custom_use_solver)) diff --git a/examples/tutorial_gridsearch.py b/examples/tutorial_gridsearch.py index a0d872a..cbead25 100644 --- a/examples/tutorial_gridsearch.py +++ b/examples/tutorial_gridsearch.py @@ -1,131 +1,129 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE -# -# Author: Sven Wanner (s.wanner@dkfz.de) # In this tutorial we solve an optimization problem using the GridsearchSolver # Gridsearch is very inefficient a Randomsearch might most of the time be the # better choice. # import the HyppopyProject class keeping track of inputs from hyppopy.HyppopyProject import HyppopyProject # import the GridsearchSolver classes from hyppopy.solver.GridsearchSolver import GridsearchSolver # import the Blackboxfunction class wrapping your problem for Hyppopy from hyppopy.BlackboxFunction import BlackboxFunction # To configure the GridsearchSolver we only need the hyperparameter section. Another # difference to the other solvers is that we need to define a gridsampling in addition # to the range: 'data': [0, 1, 100] which means sampling the space from 0 to 1 in 100 # intervals. Gridsearch also supports categorical, uniform, normal and lognormal sampling config = { "hyperparameter": { "C": { "domain": "uniform", "data": [0.0001, 20, 20], "type": "float" }, "gamma": { "domain": "uniform", "data": [0.0001, 20.0, 20], "type": "float" }, "kernel": { "domain": "categorical", "data": ["linear", "sigmoid", "poly", "rbf"], "type": "str" } }, "settings": { "solver": {}, "custom": {} }} # When creating a HyppopyProject instance we # pass the config dictionary to the constructor. project = HyppopyProject(config=config) # Hyppopy offers a class called BlackboxFunction to wrap your problem for Hyppopy. # The function signature is as follows: # BlackboxFunction(blackbox_func=None, # dataloader_func=None, # preprocess_func=None, # callback_func=None, # data=None, # **kwargs) # # Means we can set a couple of function pointers, a data object and an arbitrary number of custom parameter via kwargs. # # - blackbox_func: a function pointer to the actual, user defined, blackbox function that is computing our loss # - dataloader_func: a function pointer to a function handling the dataloading # - preprocess_func: a function pointer to a function automatically executed before starting the optimization process # - callback_func: a function pointer to a function that is called after each iteration with the trail object as input # - data: setting data can be done via dataloader_func or directly # - kwargs are passed to all functions above and thus can be used for parameter sharing between the functions # # (more details see in the documentation) # # Below we demonstrate the usage of all the above by defining a my_dataloader_function which in fact only grabs the # iris dataset from sklearn and returns it. A my_preprocess_function which also does nothing useful here but # demonstrating that a custom parameter can be set via kwargs and used in all of our functions when called within # Hyppopy. The my_callback_function gets as input the dictionary containing the state of the iteration and thus can be # used to access the current state of each solver iteration. Finally we define the actual loss_function # my_loss_function, which gets as input a data object and params. Both parameter are fixed, the first is defined by # the user depending on what is dataloader returns or the data object set in the constructor, the second is a dictionary # with a sample of your hyperparameter space which content is in the choice of the solver. from sklearn.svm import SVC from sklearn.datasets import load_iris from sklearn.model_selection import cross_val_score def my_dataloader_function(**kwargs): print("Dataloading...") iris_data = load_iris() return [iris_data.data, iris_data.target] def my_callback_function(**kwargs): print("\r{}".format(kwargs), end="") def my_loss_function(data, params): clf = SVC(**params) return -cross_val_score(estimator=clf, X=data[0], y=data[1], cv=3).mean() # We now create the BlackboxFunction object and pass all function pointers defined above, # as well as 2 dummy parameter (my_preproc_param, my_dataloader_input) for demonstration purposes. blackbox = BlackboxFunction(blackbox_func=my_loss_function, dataloader_func=my_dataloader_function, callback_func=my_callback_function) # create a solver instance solver = GridsearchSolver(project) # pass the loss function to the solver solver.blackbox = blackbox # run the solver solver.run() # get the result via get_result() which returns a pandas dataframe # containing the complete history and a dict best containing the # best parameter set. df, best = solver.get_results() print("\n") print("*"*100) print("Best Parameter Set:\n{}".format(best)) print("*"*100) diff --git a/examples/tutorial_hyppopyprojectclass.py b/examples/tutorial_hyppopyprojectclass.py index 66d285f..704674f 100644 --- a/examples/tutorial_hyppopyprojectclass.py +++ b/examples/tutorial_hyppopyprojectclass.py @@ -1,71 +1,69 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE -# -# Author: Sven Wanner (s.wanner@dkfz.de) # In this tutorial we demonstrate the HyppopyProject class usage # import the HyppopyProject class from hyppopy.HyppopyProject import HyppopyProject # To configure a solver we need to instanciate a HyppopyProject class. # This class can be configured using a nested dict. This dict has two # obligatory sections, hyperparameter and settings. A hyperparameter # is described using a dict containing a section, data and type field # and thus the hyperparameter section is a collection of hyperparameter # dicts. The settings section keeps solver settings. These might depend # on the solver used and need to be checked for each. E.g. Randomsearch, # Hyperopt and Optunity need a solver setting max_iterations, the Grid- # searchSolver don't. config = { "hyperparameter": { "C": { "domain": "uniform", "data": [0.0001, 20], "type": "float" }, "gamma": { "domain": "uniform", "data": [0.0001, 20.0], "type": "float" }, "kernel": { "domain": "categorical", "data": ["linear", "sigmoid", "poly", "rbf"], "type": "str" } }, "settings": { "solver": { "max_iterations": 500 }, "custom": {} }} # When creating a HyppopyProject instance we # pass the config dictionary to the constructor. project = HyppopyProject(config=config) # When building the project programmatically we can also use the methods # add_hyperparameter and add_settings project.clear() project.add_hyperparameter(name="C", domain="uniform", data=[0.0001, 20], dtype="float") project.add_hyperparameter(name="kernel", domain="categorical", data=["linear", "sigmoid"], dtype="str") project.add_settings(section="solver", name="max_iterations", value=500) # The custom section can be used freely project.add_settings(section="custom", name="my_var", value=10) # Settings are automatically transformed to member variables of the project class with the section as prefix if project.solver_max_iterations < 1000 and project.custom_my_var == 10: print("Project configured!") diff --git a/examples/tutorial_multisolver.py b/examples/tutorial_multisolver.py index 1dd4637..3d049f1 100644 --- a/examples/tutorial_multisolver.py +++ b/examples/tutorial_multisolver.py @@ -1,189 +1,188 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE -# -# Author: Sven Wanner (s.wanner@dkfz.de) # In this tutorial we solve an optimization problem using the Hyperopt Solver (http://hyperopt.github.io/hyperopt/). # Hyperopt uses a Baysian - Tree Parzen Estimator - Optimization approach, which means that each iteration computes a # new function value of the blackbox, interpolates a guess for the whole energy function and predicts a point to # compute the next function value at. This next point is not necessarily a "better" value, it's only the value with # the highest uncertainty for the function interpolation. # # See a visual explanation e.g. here (http://philipperemy.github.io/visualization/) # import the HyppopyProject class keeping track of inputs from hyppopy.HyppopyProject import HyppopyProject # import the SolverPool singleton class from hyppopy.SolverPool import SolverPool # import the Blackboxfunction class wrapping your problem for Hyppopy from hyppopy.BlackboxFunction import BlackboxFunction # Next step is defining the problem space and all settings Hyppopy needs to optimize your problem. # The config is a simple nested dictionary with two obligatory main sections, hyperparameter and settings. # The hyperparameter section defines your searchspace. Each hyperparameter is again a dictionary with: # # - a domain ['categorical', 'uniform', 'normal', 'loguniform'] # - the domain data [left bound, right bound] and # - a type of your domain ['str', 'int', 'float'] # # The settings section has two subcategories, solver and custom. The first contains settings for the solver, # here 'max_iterations' - is the maximum number of iteration. # # The custom section allows defining custom parameter. An entry here is transformed to a member variable of the # HyppopyProject class. These can be useful when implementing new solver classes or for control your hyppopy script. # Here we use it as a solver switch to control the usage of our solver via the config. This means with the script # below your can try out every solver by changing use_solver to 'optunity', 'randomsearch', 'gridsearch',... # It can be used like so: project.custom_use_plugin (see below) If using the gridsearch solver, max_iterations is # ignored, instead each hyperparameter must specifiy a number of samples additionally to the range like so: # 'data': [0, 1, 100] which means sampling the space from 0 to 1 in 100 intervals. config = { "hyperparameter": { "C": { "domain": "uniform", "data": [0.0001, 20], "type": "float" }, "gamma": { "domain": "uniform", "data": [0.0001, 20.0], "type": "float" }, "kernel": { "domain": "categorical", "data": ["linear", "sigmoid", "poly", "rbf"], "type": "str" }, "decision_function_shape": { "domain": "categorical", "data": ["ovo", "ovr"], "type": "str" } }, "settings": { "solver": { - "max_iterations": 200 + "max_iterations": 500 }, "custom": { "use_solver": "hyperopt" } }} # When creating a HyppopyProject instance we # pass the config dictionary to the constructor. project = HyppopyProject(config=config) # demonstration of the custom parameter access print("-"*30) print("max_iterations:\t{}".format(project.solver_max_iterations)) print("solver chosen -> {}".format(project.custom_use_solver)) print("-"*30) # The BlackboxFunction signature is as follows: # BlackboxFunction(blackbox_func=None, # dataloader_func=None, # preprocess_func=None, # callback_func=None, # data=None, # **kwargs) # # - blackbox_func: a function pointer to the users loss function # - dataloader_func: a function pointer for handling dataloading. The function is called once before # optimizing. What it returns is passed as first argument to your loss functions # data argument. # - preprocess_func: a function pointer for data preprocessing. The function is called once before # optimizing and gets via kwargs['data'] the raw data object set directly or returned # from dataloader_func. What this function returns is then what is passed as first # argument to your loss function. # - callback_func: a function pointer called after each iteration. The input kwargs is a dictionary # keeping the parameters used in this iteration, the 'iteration' index, the 'loss' # and the 'status'. The function in this example is used for realtime printing it's # input but can also be used for realtime visualization. # - data: if not done via dataloader_func one can set a raw_data object directly # - kwargs: dict that whose content is passed to all functions above. from sklearn.svm import SVC from sklearn.datasets import load_iris from sklearn.model_selection import cross_val_score def my_dataloader_function(**kwargs): print("Dataloading...") # kwargs['params'] allows accessing additional parameter passed, see below my_preproc_param, my_dataloader_input. print("my loading argument: {}".format(kwargs['params']['my_dataloader_input'])) iris_data = load_iris() return [iris_data.data, iris_data.target] def my_preprocess_function(**kwargs): print("Preprocessing...") # kwargs['data'] allows accessing the input data print("data:", kwargs['data'][0].shape, kwargs['data'][1].shape) # kwargs['params'] allows accessing additional parameter passed, see below my_preproc_param, my_dataloader_input. print("kwargs['params']['my_preproc_param']={}".format(kwargs['params']['my_preproc_param']), "\n") # if the preprocessing function returns something, # the input data will be replaced with the data returned by this function. x = kwargs['data'][0] y = kwargs['data'][1] for i in range(x.shape[0]): x[i, :] += kwargs['params']['my_preproc_param'] return [x, y] def my_callback_function(**kwargs): print("\r{}".format(kwargs), end="") def my_loss_function(data, params): clf = SVC(**params) return -cross_val_score(estimator=clf, X=data[0], y=data[1], cv=3).mean() # We now create the BlackboxFunction object and pass all function pointers defined above, # as well as 2 dummy parameter (my_preproc_param, my_dataloader_input) for demonstration purposes. blackbox = BlackboxFunction(blackbox_func=my_loss_function, dataloader_func=my_dataloader_function, preprocess_func=my_preprocess_function, callback_func=my_callback_function, my_preproc_param=1, my_dataloader_input='could/be/a/path') # Last step, is we use our SolverPool which automatically returns the correct solver. # There are multiple ways to get the desired solver from the solver pool. # 1. solver = SolverPool.get('hyperopt') # solver.project = project # 2. solver = SolverPool.get('hyperopt', project) # 3. The SolverPool will look for the field 'use_solver' in the project instance, if # it is present it will be used to specify the solver so that in this case it is enough # to pass the project instance. solver = SolverPool.get(project=project) # Give the solver your blackbox and run it. After execution we can get the result # via get_result() which returns a pandas dataframe containing the complete history # The dict best contains the best parameter set. solver.blackbox = blackbox +#solver.start_viewer() solver.run() df, best = solver.get_results() print("\n") print("*"*100) print("Best Parameter Set:\n{}".format(best)) print("*"*100) diff --git a/examples/tutorial_simple.py b/examples/tutorial_simple.py index 6e24fae..43dead1 100644 --- a/examples/tutorial_simple.py +++ b/examples/tutorial_simple.py @@ -1,86 +1,84 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE -# -# Author: Sven Wanner (s.wanner@dkfz.de) # A hyppopy minimal example optimizing a simple demo function f(x,y) = x**2+y**2 # import the HyppopyProject class keeping track of inputs from hyppopy.HyppopyProject import HyppopyProject # import the HyperoptSolver class from hyppopy.solver.HyperoptSolver import HyperoptSolver # To configure the Hyppopy solver we use a simple nested dictionary with two obligatory main sections, # hyperparameter and settings. The hyperparameter section defines your searchspace. Each hyperparameter # is again a dictionary with: # # - a domain ['categorical', 'uniform', 'normal', 'loguniform'] # - the domain data [left bound, right bound] and # - a type of your domain ['str', 'int', 'float'] # # The settings section has two subcategories, solver and custom. The first contains settings for the solver, # here 'max_iterations' - is the maximum number of iteration. # # The custom section allows defining custom parameter. An entry here is transformed to a member variable of the # HyppopyProject class. These can be useful when implementing new solver classes or for control your hyppopy script. # Here we use it as a solver switch to control the usage of our solver via the config. This means with the script # below your can try out every solver by changing use_solver to 'optunity', 'randomsearch', 'gridsearch',... # It can be used like so: project.custom_use_plugin (see below) If using the gridsearch solver, max_iterations is # ignored, instead each hyperparameter must specifiy a number of samples additionally to the range like so: # 'data': [0, 1, 100] which means sampling the space from 0 to 1 in 100 intervals. config = { "hyperparameter": { "x": { "domain": "normal", "data": [-10.0, 10.0], "type": "float" }, "y": { "domain": "uniform", "data": [-10.0, 10.0], "type": "float" } }, "settings": { "solver": { "max_iterations": 500 }, "custom": {} }} # When creating a HyppopyProject instance we # pass the config dictionary to the constructor. project = HyppopyProject(config=config) # The user defined loss function def my_loss_function(x, y): return x**2+y**2 # create a solver instance solver = HyperoptSolver(project) # pass the loss function to the solver solver.blackbox = my_loss_function # run the solver solver.run() df, best = solver.get_results() print("\n") print("*"*100) print("Best Parameter Set:\n{}".format(best)) print("*"*100) diff --git a/hyppopy/BlackboxFunction.py b/hyppopy/BlackboxFunction.py index 8fce552..1348d4f 100644 --- a/hyppopy/BlackboxFunction.py +++ b/hyppopy/BlackboxFunction.py @@ -1,98 +1,96 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE -# -# Author: Sven Wanner (s.wanner@dkfz.de) import os import logging import functools from hyppopy.globals import DEBUGLEVEL LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) def default_kwargs(**defaultKwargs): def actual_decorator(fn): @functools.wraps(fn) def g(*args, **kwargs): defaultKwargs.update(kwargs) return fn(*args, **defaultKwargs) return g return actual_decorator class BlackboxFunction(object): @default_kwargs(blackbox_func=None, dataloader_func=None, preprocess_func=None, callback_func=None, data=None) def __init__(self, **kwargs): self._blackbox_func = None self._preprocess_func = None self._dataloader_func = None self._callback_func = None self._raw_data = None self._data = None self.setup(kwargs) def __call__(self, **kwargs): return self.blackbox_func(self.data, kwargs) def setup(self, kwargs): self._blackbox_func = kwargs['blackbox_func'] self._preprocess_func = kwargs['preprocess_func'] self._dataloader_func = kwargs['dataloader_func'] self._callback_func = kwargs['callback_func'] self._raw_data = kwargs['data'] self._data = self._raw_data del kwargs['blackbox_func'] del kwargs['preprocess_func'] del kwargs['dataloader_func'] del kwargs['data'] params = kwargs if self.dataloader_func is not None: self._raw_data = self.dataloader_func(params=params) assert self._raw_data is not None, "Missing data exception!" assert self.blackbox_func is not None, "Missing blackbox fucntion exception!" if self.preprocess_func is not None: result = self.preprocess_func(data=self._raw_data, params=params) if result is not None: self._data = result else: self._data = self._raw_data else: self._data = self._raw_data @property def blackbox_func(self): return self._blackbox_func @property def preprocess_func(self): return self._preprocess_func @property def dataloader_func(self): return self._dataloader_func @property def callback_func(self): return self._callback_func @property def raw_data(self): return self._raw_data @property def data(self): return self._data diff --git a/hyppopy/HyppopyProject.py b/hyppopy/HyppopyProject.py index 19409fa..ef5f22b 100644 --- a/hyppopy/HyppopyProject.py +++ b/hyppopy/HyppopyProject.py @@ -1,103 +1,101 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE -# -# Author: Sven Wanner (s.wanner@dkfz.de) import warnings from hyppopy.globals import * LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) class HyppopyProject(object): def __init__(self, config=None): self._hyperparameter = {} self._settings = {} self._extmembers = [] if config is not None: self.set_config(config) def clear(self): self._hyperparameter = {} self._settings = {} for added in self._extmembers: if added in self.__dict__.keys(): del self.__dict__[added] self._extmembers = [] def set_config(self, config): self.clear() assert isinstance(config, dict), "Input Error, config of type {} not supported!".format(type(config)) assert HYPERPARAMETERPATH in config.keys(), "Missing hyperparameter section in config dict" #assert SETTINGSPATH in config.keys(), "Missing settings section in config dict" if not SETTINGSPATH in config.keys(): config[SETTINGSPATH] = {"solver": {"max_iterations": DEFAULTITERATIONS}} msg = "config dict had no section {0}/solver/max_iterations, set default value: {1}".format(SETTINGSPATH, DEFAULTITERATIONS) warnings.warn(msg) LOG.warning(msg) self._hyperparameter = config[HYPERPARAMETERPATH] self._settings = config[SETTINGSPATH] self.parse_members() def add_hyperparameter(self, name, domain, data, dtype): assert isinstance(name, str), "precondition violation, name of type {} not allowed, expect str!".format(type(name)) assert isinstance(domain, str), "precondition violation, domain of type {} not allowed, expect str!".format(type(domain)) assert domain in SUPPORTED_DOMAINS, "domain {} not supported, expect {}!".format(domain, SUPPORTED_DOMAINS) assert isinstance(data, list) or isinstance(data, tuple), "precondition violation, data of type {} not allowed, expect list or tuple!".format(type(data)) if domain != "categorical": assert len(data) == 3 or len(data) == 2, "precondition violation, data must be a list of len 2 or 3" assert isinstance(dtype, str), "precondition violation, dtype of type {} not allowed, expect str!".format(type(dtype)) assert dtype in SUPPORTED_DTYPES, "precondition violation, dtype {} not supported, expect {}!".format(dtype, SUPPORTED_DTYPES) self._hyperparameter[name] = {"domain": domain, "data": data, "type": dtype} def add_settings(self, section, name, value): assert isinstance(section, str), "precondition violation, section of type {} not allowed, expect str!".format(type(section)) assert isinstance(name, str), "precondition violation, name of type {} not allowed, expect str!".format(type(name)) if section not in self._settings.keys(): self._settings[section] = {} self._settings[section][name] = value self.parse_members() def parse_members(self): for section_name, content in self.settings.items(): for name, value in content.items(): member_name = section_name + "_" + name if member_name not in self._extmembers: setattr(self, member_name, value) self._extmembers.append(member_name) else: self.__dict__[member_name] = value def get_typeof(self, hyperparametername): if not hyperparametername in self.hyperparameter.keys(): return None dtype = self.hyperparameter[hyperparametername]["type"] if dtype == 'str': return str if dtype == 'int': return int if dtype == 'float': return float @property def hyperparameter(self): return self._hyperparameter @property def settings(self): return self._settings diff --git a/hyppopy/ProjectManager.py b/hyppopy/ProjectManager.py index 4257bf8..0d072dd 100644 --- a/hyppopy/ProjectManager.py +++ b/hyppopy/ProjectManager.py @@ -1,69 +1,67 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE -# -# Author: Sven Wanner (s.wanner@dkfz.de) from .Singleton import * import os import logging from hyppopy.HyppopyProject import HyppopyProject from hyppopy.globals import DEBUGLEVEL LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) @singleton_object class ProjectManager(metaclass=Singleton): def __init__(self): self._current_project = None self._projects = {} def clear_all(self): pass def new_project(self, name="HyppopyProject", config=None): if name in self._projects.keys(): name = self.check_projectname(name) self._projects[name] = HyppopyProject(config) self._current_project = self._projects[name] return self._current_project def check_projectname(self, name): split = name.split(".") if len(split) == 0: return split[0] + "." + str(0).zfill(3) else: try: number = int(split[-1]) del split[-1] except: number = 0 return '.'.join(split) + "." + str(number).zfill(3) def get_current(self): if self._current_project is None: self.new_project() return self._current_project def get_project(self, name): if name in self._projects.keys(): self._current_project = self._projects[name] return self.get_current() return self.new_project(name) def get_projectnames(self): return self._projects.keys() diff --git a/hyppopy/Singleton.py b/hyppopy/Singleton.py index 90575de..fac0bc8 100644 --- a/hyppopy/Singleton.py +++ b/hyppopy/Singleton.py @@ -1,52 +1,50 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE -# -# Author: Sven Wanner (s.wanner@dkfz.de) class Singleton(type): _instances = {} def __call__(cls, *args, **kwargs): if cls not in cls._instances: cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) return cls._instances[cls] @classmethod def __instancecheck__(mcs, instance): if instance.__class__ is mcs: return True else: return isinstance(instance.__class__, mcs) def singleton_object(cls): """Class decorator that transforms (and replaces) a class definition (which must have a Singleton metaclass) with the actual singleton object. Ensures that the resulting object can still be "instantiated" (i.e., called), returning the same object. Also ensures the object can be pickled, is hashable, and has the correct string representation (the name of the singleton) """ assert isinstance(cls, Singleton), cls.__name__ + " must use Singleton metaclass" def self_instantiate(self): return self cls.__call__ = self_instantiate cls.__hash__ = lambda self: hash(cls) cls.__repr__ = lambda self: cls.__name__ cls.__reduce__ = lambda self: cls.__name__ obj = cls() obj.__name__ = cls.__name__ return obj diff --git a/hyppopy/Solver/BayesOptSolver.py b/hyppopy/Solver/BayesOptSolver.py index 338cb8b..5a58bdf 100644 --- a/hyppopy/Solver/BayesOptSolver.py +++ b/hyppopy/Solver/BayesOptSolver.py @@ -1,84 +1,82 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE -# -# Author: Sven Wanner (s.wanner@dkfz.de) import os import logging import warnings import numpy as np from pprint import pformat from hyperopt import Trials from bayes_opt import BayesianOptimization from hyppopy.globals import DEBUGLEVEL from hyppopy.solver.HyppopySolver import HyppopySolver LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) class BayesOptSolver(HyppopySolver): def __init__(self, project=None): HyppopySolver.__init__(self, project) self._searchspace = None self._idx = None def reformat_parameter(self, params): out_params = {} for name, value in params.items(): if self._searchspace[name]["domain"] == "categorical": out_params[name] = self._searchspace[name]["data"][int(np.round(value))] else: if self._searchspace[name]["type"] == "int": out_params[name] = int(np.round(value)) else: out_params[name] = value return out_params def loss_function_call(self, params): params = self.reformat_parameter(params) for key in params.keys(): if self.project.get_typeof(key) is int: params[key] = int(round(params[key])) return self.blackbox(**params) def execute_solver(self, searchspace): LOG.debug("execute_solver using solution space:\n\n\t{}\n".format(pformat(searchspace))) self.trials = Trials() self._idx = 0 try: optimizer = BayesianOptimization(f=self.loss_function, pbounds=searchspace, verbose=0) optimizer.maximize(init_points=2, n_iter=self.max_iterations) self.best = self.reformat_parameter(optimizer.max["params"]) except Exception as e: LOG.error("internal error in bayes_opt maximize occured. {}".format(e)) raise BrokenPipeError("internal error in bayes_opt maximize occured. {}".format(e)) def convert_searchspace(self, hyperparameter): LOG.debug("convert input parameter\n\n\t{}\n".format(pformat(hyperparameter))) self._searchspace = hyperparameter pbounds = {} for name, param in hyperparameter.items(): if param["domain"] != "categorical": if param["domain"] != "uniform": msg = "Warning: BayesOpt cannot handle {} domain. Only uniform and categorical domains are supported!".format( param["domain"]) warnings.warn(msg) LOG.warning(msg) pbounds[name] = (param["data"][0], param["data"][1]) else: pbounds[name] = (0, len(param["data"])-1) return pbounds diff --git a/hyppopy/Solver/GridsearchSolver.py b/hyppopy/Solver/GridsearchSolver.py index d0a6604..272482a 100644 --- a/hyppopy/Solver/GridsearchSolver.py +++ b/hyppopy/Solver/GridsearchSolver.py @@ -1,182 +1,180 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE -# -# Author: Sven Wanner (s.wanner@dkfz.de) import os import logging import numpy as np from pprint import pformat from scipy.stats import norm from itertools import product from hyppopy.globals import DEBUGLEVEL from hyppopy.solver.HyppopySolver import HyppopySolver LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) def get_uniform_axis_sample(a, b, N, dtype): """ returns a uniform sample x(n) in the range [a,b] sampled at N pojnts :param a: left value range bound :param b: right value range bound :param N: discretization of intervall [a,b] :param dtype: data type :return: [list] axis range """ assert a < b, "condition a < b violated!" assert isinstance(N, int), "condition N of type int violated!" assert isinstance(dtype, str), "condition type of type str violated!" if dtype == "int": return list(np.linspace(a, b, N).astype(int)) elif dtype == "float" or dtype == "double": return list(np.linspace(a, b, N)) else: raise AssertionError("dtype {} not supported for uniform sampling!".format(dtype)) def get_norm_cdf(N): """ returns a normed gaussian cdf (range [0,1]) with N sampling points :param N: sampling points :return: [ndarray] gaussian cdf function values """ assert isinstance(N, int), "condition N of type int violated!" even = True if N % 2 != 0: N -= 1 even = False N = int(N/2) sigma = 1/3 x = np.linspace(0, 1, N) y1 = norm.cdf(x, loc=0, scale=sigma)-0.5 if not even: y1 = np.append(y1, [0.5]) y2 = 1-(norm.cdf(x, loc=0, scale=sigma)-0.5) y2 = np.flip(y2, axis=0) y = np.concatenate((y1, y2), axis=0) return y def get_gaussian_axis_sample(a, b, N, dtype): """ returns a function value f(n) where f is a gaussian cdf in range [a, b] and N sampling points :param a: left value range bound :param b: right value range bound :param N: discretization of intervall [a,b] :param dtype: data type :return: [list] axis range """ assert a < b, "condition a < b violated!" assert isinstance(N, int), "condition N of type int violated!" assert isinstance(dtype, str), "condition type of type str violated!" data = [] for n in range(N): x = a + get_norm_cdf(N)[n]*(b-a) if dtype == "int": data.append(int(x)) elif dtype == "float" or dtype == "double": data.append(x) else: raise AssertionError("dtype {} not supported for uniform sampling!".format(dtype)) return data def get_logarithmic_axis_sample(a, b, N, dtype): """ returns a function value f(n) where f is logarithmic function e^x sampling the exponent range [log(a), log(b)] linear at N sampling points. The function values returned are in the range [a, b]. :param a: left value range bound :param b: right value range bound :param N: discretization of intervall [a,b] :param dtype: data type :return: [list] axis range """ assert a < b, "condition a < b violated!" assert a > 0, "condition a > 0 violated!" assert isinstance(N, int), "condition N of type int violated!" assert isinstance(dtype, str), "condition type of type str violated!" # convert input range into exponent range lexp = np.log(a) rexp = np.log(b) exp_range = np.linspace(lexp, rexp, N) data = [] for n in range(exp_range.shape[0]): x = np.exp(exp_range[n]) if dtype == "int": data.append(int(x)) elif dtype == "float" or dtype == "double": data.append(x) else: raise AssertionError("dtype {} not supported for uniform sampling!".format(dtype)) return data class GridsearchSolver(HyppopySolver): """ The GridsearchSolver class implements a gridsearch optimization. The gridsearch supports categorical, uniform, normal and loguniform sampling. To use the GridsearchSolver, besides a range, one must specifiy the number of samples in the domain, e.g. 'data': [0, 1, 100] """ def __init__(self, project=None): HyppopySolver.__init__(self, project) self._has_maxiteration_field = False def loss_function_call(self, params): loss = self.blackbox(**params) if loss is None: return np.nan return loss def execute_solver(self, searchspace): for x in product(*searchspace[1]): params = {} for name, value in zip(searchspace[0], x): params[name] = value try: self.loss_function(**params) except Exception as e: msg = "internal error in randomsearch execute_solver occured. {}".format(e) LOG.error(msg) raise BrokenPipeError(msg) self.best = self._trials.argmin def convert_searchspace(self, hyperparameter): """ the function converts the standard parameter input into a range list depending on the domain. These rangelists are later used with itertools product to create a paramater space sample of each combination. :param hyperparameter: [dict] hyperparameter space :return: [list] name and range for each parameter space axis """ LOG.debug("convert input parameter\n\n\t{}\n".format(pformat(hyperparameter))) searchspace = [[], []] for name, param in hyperparameter.items(): if param["domain"] == "categorical": searchspace[0].append(name) searchspace[1].append(param["data"]) elif param["domain"] == "uniform": searchspace[0].append(name) searchspace[1].append(get_uniform_axis_sample(param["data"][0], param["data"][1], param["data"][2], param["type"])) elif param["domain"] == "normal": searchspace[0].append(name) searchspace[1].append(get_gaussian_axis_sample(param["data"][0], param["data"][1], param["data"][2], param["type"])) elif param["domain"] == "loguniform": searchspace[0].append(name) searchspace[1].append(get_logarithmic_axis_sample(param["data"][0], param["data"][1], param["data"][2], param["type"])) return searchspace diff --git a/hyppopy/Solver/HyperoptSolver.py b/hyppopy/Solver/HyperoptSolver.py index b9b91ff..19ec8ee 100644 --- a/hyppopy/Solver/HyperoptSolver.py +++ b/hyppopy/Solver/HyperoptSolver.py @@ -1,158 +1,164 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE -# -# Author: Sven Wanner (s.wanner@dkfz.de) import os import copy import logging import numpy as np from pprint import pformat from hyperopt import fmin, tpe, hp, STATUS_OK, STATUS_FAIL, Trials from hyppopy.globals import DEBUGLEVEL from hyppopy.solver.HyppopySolver import HyppopySolver from hyppopy.BlackboxFunction import BlackboxFunction LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) class HyperoptSolver(HyppopySolver): def __init__(self, project=None): HyppopySolver.__init__(self, project) self._searchspace = None def loss_function(self, params): for name, p in self._searchspace.items(): if p["domain"] != "categorical": if params[name] < p["data"][0]: params[name] = p["data"][0] if params[name] > p["data"][1]: params[name] = p["data"][1] status = STATUS_FAIL try: loss = self.blackbox(**params) if loss is not None: status = STATUS_OK else: loss = 1e9 except Exception as e: LOG.error("execution of self.blackbox(**params) failed due to:\n {}".format(e)) status = STATUS_FAIL loss = 1e9 + cbd = copy.deepcopy(params) + cbd['iterations'] = self._trials.trials[-1]['tid'] + 1 + cbd['loss'] = loss + cbd['status'] = status + cbd['book_time'] = self._trials.trials[-1]['book_time'] + cbd['refresh_time'] = self._trials.trials[-1]['refresh_time'] if isinstance(self.blackbox, BlackboxFunction) and self.blackbox.callback_func is not None: - cbd = copy.deepcopy(params) - cbd['iterations'] = self._trials.trials[-1]['tid'] + 1 - cbd['loss'] = loss - cbd['status'] = status + # cbd = copy.deepcopy(params) + # cbd['iterations'] = self._trials.trials[-1]['tid'] + 1 + # cbd['loss'] = loss + # cbd['status'] = status self.blackbox.callback_func(**cbd) + if self._visdom_viewer is not None: + self._visdom_viewer.update(cbd) return {'loss': loss, 'status': status} def execute_solver(self, searchspace): LOG.debug("execute_solver using solution space:\n\n\t{}\n".format(pformat(searchspace))) self.trials = Trials() try: self.best = fmin(fn=self.loss_function, space=searchspace, algo=tpe.suggest, max_evals=self.max_iterations, trials=self.trials) except Exception as e: msg = "internal error in hyperopt.fmin occured. {}".format(e) LOG.error(msg) raise BrokenPipeError(msg) def convert_searchspace(self, hyperparameter): self._searchspace = hyperparameter solution_space = {} for name, content in hyperparameter.items(): param_settings = {'name': name} for key, value in content.items(): if key == 'domain': param_settings['domain'] = value elif key == 'data': param_settings['data'] = value elif key == 'type': param_settings['dtype'] = value solution_space[name] = self.convert(param_settings) return solution_space def convert(self, param_settings): name = param_settings["name"] domain = param_settings["domain"] dtype = param_settings["dtype"] data = param_settings["data"] assert isinstance(data, list), "precondition violation. data of type {} not allowed!".format(type(data)) assert len(data) >= 2, "precondition violation, data must be of length 2, [left_bound, right_bound]" assert isinstance(domain, str), "precondition violation. domain of type {} not allowed!".format(type(domain)) assert isinstance(dtype, str), "precondition violation. dtype of type {} not allowed!".format(type(dtype)) if domain == "uniform": if dtype == "float" or dtype == "double": return hp.uniform(name, data[0], data[1]) elif dtype == "int": data = list(np.arange(int(data[0]), int(data[1] + 1))) return hp.choice(name, data) else: msg = "cannot convert the type {} in domain {}".format(dtype, domain) LOG.error(msg) raise LookupError(msg) elif domain == "loguniform": if dtype == "float" or dtype == "double": if data[0] == 0: data[0] += 1e-23 assert data[0] > 0, "precondition Violation, a < 0!" assert data[0] < data[1], "precondition Violation, a > b!" assert data[1] > 0, "precondition Violation, b < 0!" lexp = np.log(data[0]) rexp = np.log(data[1]) assert lexp is not np.nan, "precondition violation, left bound input error, results in nan!" assert rexp is not np.nan, "precondition violation, right bound input error, results in nan!" return hp.loguniform(name, lexp, rexp) else: msg = "cannot convert the type {} in domain {}".format(dtype, domain) LOG.error(msg) raise LookupError(msg) elif domain == "normal": if dtype == "float" or dtype == "double": mu = (data[1] - data[0]) / 2.0 sigma = mu / 3 return hp.normal(name, data[0] + mu, sigma) else: msg = "cannot convert the type {} in domain {}".format(dtype, domain) LOG.error(msg) raise LookupError(msg) elif domain == "categorical": if dtype == 'str': return hp.choice(name, data) elif dtype == 'bool': data = [] for elem in data: if elem == "true" or elem == "True" or elem == 1 or elem == "1": data.append(True) elif elem == "false" or elem == "False" or elem == 0 or elem == "0": data.append(False) else: msg = "cannot convert the type {} in domain {}, unknown bool type value".format(dtype, domain) LOG.error(msg) raise LookupError(msg) return hp.choice(name, data) else: msg = "Precondition violation, domain named {} not available!".format(domain) LOG.error(msg) raise IOError(msg) diff --git a/hyppopy/Solver/HyppopySolver.py b/hyppopy/Solver/HyppopySolver.py index 7d59869..864985d 100644 --- a/hyppopy/Solver/HyppopySolver.py +++ b/hyppopy/Solver/HyppopySolver.py @@ -1,329 +1,342 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE -# -# Author: Sven Wanner (s.wanner@dkfz.de) import abc import os import copy import types import logging import datetime import numpy as np import pandas as pd from hyperopt import Trials from hyppopy.globals import DEBUGLEVEL +from hyppopy.VisdomViewer import VisdomViewer from hyppopy.HyppopyProject import HyppopyProject from hyppopy.BlackboxFunction import BlackboxFunction from hyppopy.VirtualFunction import VirtualFunction from hyppopy.globals import DEBUGLEVEL, DEFAULTITERATIONS LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) class HyppopySolver(object): """ The HyppopySolver class is the base class for all solver addons. It defines virtual functions a child class has to implement to deal with the front-end communication, orchestrating the optimization process and ensuring a proper process information storing. The key idea is that the HyppopySolver class defines an interface to configure and run an object instance of itself independently from the concrete solver lib used to optimize in the background. To achieve this goal an addon developer needs to implement the abstract methods 'convert_searchspace', 'execute_solver' and 'loss_function_call'. These methods abstract the peculiarities of the solver libs to offer, on the user side, a simple and consistent parameter space configuration and optimization procedure. The method 'convert_searchspace' transforms the hyppopy parameter space description into the solver lib specific description. The method loss_function_call is used to handle solver lib specifics of calling the actual blackbox function and execute_solver is executed when the run method is invoked und takes care of calling the solver lib solving routine. """ def __init__(self, project=None): self._idx = None self._best = None self._trials = None self._blackbox = None self._max_iterations = None self._project = project self._total_duration = None self._solver_overhead = None self._time_per_iteration = None self._accumulated_blackbox_time = None self._has_maxiteration_field = True + self._visdom_viewer = None @abc.abstractmethod def convert_searchspace(self, hyperparameter): """ This function gets the unified hyppopy-like parameterspace description as input and, if necessary, should convert it into a solver lib specific format. The function is invoked when run is called and what it returns is passed as searchspace argument to the function execute_solver. :param hyperparameter: [dict] nested parameter description dict e.g. {'name': {'domain':'uniform', 'data':[0,1], 'type':'float'}, ...} :return: [object] converted hyperparameter space """ raise NotImplementedError('users must define convert_searchspace to use this class') @abc.abstractmethod def execute_solver(self, searchspace): """ This function is called immediatly after convert_searchspace and get the output of the latter as input. It's purpose is to call the solver libs main optimization function. :param searchspace: converted hyperparameter space """ raise NotImplementedError('users must define execute_solver to use this class') @abc.abstractmethod def loss_function_call(self, params): """ This function is called within the function loss_function and encapsulates the actual blackbox function call in each iteration. The function loss_function takes care of the iteration driving and reporting, but each solver lib might need some special treatment between the parameter set selection and the calling of the actual blackbox function, e.g. parameter converting. :param params: [dict] hyperparameter space sample e.g. {'p1': 0.123, 'p2': 3.87, ...} :return: [float] loss """ raise NotImplementedError('users must define convert_searchspace to use this class') def loss_function(self, **params): """ This function is called each iteration with a selected parameter set. The parameter set selection is driven by the solver lib itself. The purpose of this function is to take care of the iteration reporting and the calling of the callback_func is available. As a developer you might want to overwrite this function completely (e.g. HyperoptSolver) but then you need to take care for iteration reporting for yourself. The alternative is to only implement loss_function_call (e.g. OptunitySolver). :param params: [dict] hyperparameter space sample e.g. {'p1': 0.123, 'p2': 3.87, ...} :return: [float] loss """ self._idx += 1 vals = {} idx = {} for key, value in params.items(): vals[key] = [value] idx[key] = [self._idx] trial = {'tid': self._idx, 'result': {'loss': None, 'status': 'ok'}, 'misc': { 'tid': self._idx, 'idxs': idx, 'vals': vals }, 'book_time': datetime.datetime.now(), 'refresh_time': None } try: loss = self.loss_function_call(params) trial['result']['loss'] = loss trial['result']['status'] = 'ok' if loss == np.nan: trial['result']['status'] = 'failed' except Exception as e: LOG.error("computing loss failed due to:\n {}".format(e)) loss = np.nan trial['result']['loss'] = np.nan trial['result']['status'] = 'failed' trial['refresh_time'] = datetime.datetime.now() self._trials.trials.append(trial) + cbd = copy.deepcopy(params) + cbd['iterations'] = self._idx + cbd['loss'] = loss + cbd['status'] = trial['result']['status'] + cbd['book_time'] = trial['book_time'] + cbd['refresh_time'] = trial['refresh_time'] if isinstance(self.blackbox, BlackboxFunction) and self.blackbox.callback_func is not None: - cbd = copy.deepcopy(params) - cbd['iterations'] = self._idx - cbd['loss'] = loss - cbd['status'] = trial['result']['status'] self.blackbox.callback_func(**cbd) + if self._visdom_viewer is not None: + self._visdom_viewer.update(cbd) return loss def run(self, print_stats=True): """ This function starts the optimization process. :param print_stats: [bool] en- or disable console output """ self._idx = 0 self.trials = Trials() if self._has_maxiteration_field: if 'solver_max_iterations' not in self.project.__dict__: msg = "Missing max_iteration entry in project, use default {}!".format(DEFAULTITERATIONS) LOG.warning(msg) print("WARNING: {}".format(msg)) setattr(self.project, 'solver_max_iterations', DEFAULTITERATIONS) self._max_iterations = self.project.solver_max_iterations start_time = datetime.datetime.now() try: search_space = self.convert_searchspace(self.project.hyperparameter) except Exception as e: msg = "Failed to convert searchspace, error: {}".format(e) LOG.error(msg) raise AssertionError(msg) try: self.execute_solver(search_space) except Exception as e: msg = "Failed to execute solver, error: {}".format(e) LOG.error(msg) raise AssertionError(msg) end_time = datetime.datetime.now() dt = end_time - start_time days = divmod(dt.total_seconds(), 86400) hours = divmod(days[1], 3600) minutes = divmod(hours[1], 60) seconds = divmod(minutes[1], 1) milliseconds = divmod(seconds[1], 0.001) self._total_duration = [int(days[0]), int(hours[0]), int(minutes[0]), int(seconds[0]), int(milliseconds[0])] if print_stats: self.print_best() self.print_timestats() def get_results(self): """ This function returns a complete optimization history as pandas DataFrame and a dict with the optimal parameter set. :return: [DataFrame], [dict] history and optimal parameter set """ assert isinstance(self.trials, Trials), "precondition violation, wrong trials type! Maybe solver was not yet executed?" results = {'duration': [], 'losses': [], 'status': []} pset = self.trials.trials[0]['misc']['vals'] for p in pset.keys(): results[p] = [] for n, trial in enumerate(self.trials.trials): t1 = trial['book_time'] t2 = trial['refresh_time'] results['duration'].append((t2 - t1).microseconds / 1000.0) results['losses'].append(trial['result']['loss']) results['status'].append(trial['result']['status'] == 'ok') losses = np.array(results['losses']) results['losses'] = list(losses) pset = trial['misc']['vals'] for p in pset.items(): results[p[0]].append(p[1][0]) return pd.DataFrame.from_dict(results), self.best def print_best(self): print("\n") print("#" * 40) print("### Best Parameter Choice ###") print("#" * 40) for name, value in self.best.items(): print(" - {}\t:\t{}".format(name, value)) print("\n - number of iterations\t:\t{}".format(self.trials.trials[-1]['tid']+1)) print(" - total time\t:\t{}d:{}h:{}m:{}s:{}ms".format(self._total_duration[0], self._total_duration[1], self._total_duration[2], self._total_duration[3], self._total_duration[4])) print("#" * 40) def compute_time_statistics(self): dts = [] for trial in self._trials.trials: if 'book_time' in trial.keys() and 'refresh_time' in trial.keys(): dt = trial['refresh_time'] - trial['book_time'] dts.append(dt.total_seconds()) self._time_per_iteration = np.mean(dts) * 1e3 self._accumulated_blackbox_time = np.sum(dts) * 1e3 tmp = self.total_duration - self._accumulated_blackbox_time self._solver_overhead = int(np.round(100.0 / (self.total_duration+1e-12) * tmp)) def print_timestats(self): print("\n") print("#" * 40) print("### Timing Statistics ###") print("#" * 40) print(" - per iteration: {}ms".format(int(self.time_per_iteration*1e4)/10000)) print(" - total time: {}d:{}h:{}m:{}s:{}ms".format(self._total_duration[0], self._total_duration[1], self._total_duration[2], self._total_duration[3], self._total_duration[4])) print("#" * 40) print(" - solver overhead: {}%".format(self.solver_overhead)) + def start_viewer(self, port=8097, server="http://localhost"): + try: + self._visdom_viewer = VisdomViewer(self._project, port, server) + except Exception as e: + import warnings + warnings.warn("Failed starting VisdomViewer. Is the server running? If not start it via $visdom") + LOG.error("Failed starting VisdomViewer: {}".format(e)) + self._visdom_viewer = None + @property def project(self): return self._project @project.setter def project(self, value): if not isinstance(value, HyppopyProject): msg = "Input error, project_manager of type: {} not allowed!".format(type(value)) LOG.error(msg) raise IOError(msg) self._project = value @property def blackbox(self): return self._blackbox @blackbox.setter def blackbox(self, value): if isinstance(value, types.FunctionType) or isinstance(value, BlackboxFunction) or isinstance(value, VirtualFunction): self._blackbox = value else: self._blackbox = None msg = "Input error, blackbox of type: {} not allowed!".format(type(value)) LOG.error(msg) raise IOError(msg) @property def best(self): return self._best @best.setter def best(self, value): if not isinstance(value, dict): msg = "Input error, best of type: {} not allowed!".format(type(value)) LOG.error(msg) raise IOError(msg) self._best = value @property def trials(self): return self._trials @trials.setter def trials(self, value): self._trials = value @property def max_iterations(self): return self._max_iterations @max_iterations.setter def max_iterations(self, value): if not isinstance(value, int): msg = "Input error, max_iterations of type: {} not allowed!".format(type(value)) LOG.error(msg) raise IOError(msg) if value < 1: msg = "Precondition violation, max_iterations < 1!" LOG.error(msg) raise IOError(msg) self._max_iterations = value @property def total_duration(self): return (self._total_duration[0]*86400 + self._total_duration[1] * 3600 + self._total_duration[2] * 60 + self._total_duration[3]) * 1000 + self._total_duration[4] @property def solver_overhead(self): if self._solver_overhead is None: self.compute_time_statistics() return self._solver_overhead @property def time_per_iteration(self): if self._time_per_iteration is None: self.compute_time_statistics() return self._time_per_iteration @property def accumulated_blackbox_time(self): if self._accumulated_blackbox_time is None: self.compute_time_statistics() return self._accumulated_blackbox_time diff --git a/hyppopy/Solver/OptunaSolver.py b/hyppopy/Solver/OptunaSolver.py index 9074a31..d3d4871 100644 --- a/hyppopy/Solver/OptunaSolver.py +++ b/hyppopy/Solver/OptunaSolver.py @@ -1,82 +1,80 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE -# -# Author: Sven Wanner (s.wanner@dkfz.de) import os import optuna import logging import warnings import numpy as np from pprint import pformat from hyppopy.globals import DEBUGLEVEL from hyppopy.solver.HyppopySolver import HyppopySolver LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) class OptunaSolver(HyppopySolver): def __init__(self, project=None): HyppopySolver.__init__(self, project) self._searchspace = None def reformat_parameter(self, params): out_params = {} for name, value in params.items(): if self._searchspace[name]["domain"] == "categorical": out_params[name] = self._searchspace[name]["data"][int(np.round(value))] else: if self._searchspace[name]["type"] == "int": out_params[name] = int(np.round(value)) else: out_params[name] = value return out_params def trial_cache(self, trial): params = {} for name, param in self._searchspace.items(): if param["domain"] == "categorical": params[name] = trial.suggest_categorical(name, param["data"]) else: params[name] = trial.suggest_uniform(name, param["data"][0], param["data"][1]) return self.loss_function(**params) def loss_function_call(self, params): for key in params.keys(): if self.project.get_typeof(key) is int: params[key] = int(round(params[key])) return self.blackbox(**params) def execute_solver(self, searchspace): LOG.debug("execute_solver using solution space:\n\n\t{}\n".format(pformat(searchspace))) self._searchspace = searchspace try: study = optuna.create_study() study.optimize(self.trial_cache, n_trials=self.max_iterations) self.best = study.best_trial.params except Exception as e: LOG.error("internal error in bayes_opt maximize occured. {}".format(e)) raise BrokenPipeError("internal error in bayes_opt maximize occured. {}".format(e)) def convert_searchspace(self, hyperparameter): LOG.debug("convert input parameter\n\n\t{}\n".format(pformat(hyperparameter))) for name, param in hyperparameter.items(): if param["domain"] != "categorical" and param["domain"] != "uniform": msg = "Warning: Optuna cannot handle {} domain. Only uniform and categorical domains are supported!".format(param["domain"]) warnings.warn(msg) LOG.warning(msg) return hyperparameter diff --git a/hyppopy/Solver/OptunitySolver.py b/hyppopy/Solver/OptunitySolver.py index 7a40117..5a8b337 100644 --- a/hyppopy/Solver/OptunitySolver.py +++ b/hyppopy/Solver/OptunitySolver.py @@ -1,98 +1,96 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE -# -# Author: Sven Wanner (s.wanner@dkfz.de) import os import logging import optunity import warnings from pprint import pformat from hyppopy.globals import DEBUGLEVEL LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) from hyppopy.solver.HyppopySolver import HyppopySolver class OptunitySolver(HyppopySolver): def __init__(self, project=None): HyppopySolver.__init__(self, project) self._solver_info = None self.opt_trials = None def loss_function_call(self, params): for key in params.keys(): if self.project.get_typeof(key) is int: params[key] = int(round(params[key])) return self.blackbox(**params) def execute_solver(self, searchspace): LOG.debug("execute_solver using solution space:\n\n\t{}\n".format(pformat(searchspace))) try: self.best, _, _ = optunity.minimize_structured(f=self.loss_function, num_evals=self.max_iterations, search_space=searchspace) except Exception as e: LOG.error("internal error in optunity.minimize_structured occured. {}".format(e)) raise BrokenPipeError("internal error in optunity.minimize_structured occured. {}".format(e)) def split_categorical(self, pdict): categorical = {} uniform = {} for name, pset in pdict.items(): for key, value in pset.items(): if key == 'domain' and value == 'categorical': categorical[name] = pset elif key == 'domain': if value != 'uniform': msg = "Warning: Optunity cannot handle {} domain. Only uniform and categorical domains are supported!".format(value) warnings.warn(msg) LOG.warning(msg) uniform[name] = pset return categorical, uniform def convert_searchspace(self, hyperparameter): LOG.debug("convert input parameter\n\n\t{}\n".format(pformat(hyperparameter))) solution_space = {} # split input in categorical and non-categorical data cat, uni = self.split_categorical(hyperparameter) # build up dictionary keeping all non-categorical data uniforms = {} for key, value in uni.items(): for key2, value2 in value.items(): if key2 == 'data': if len(value2) == 3: uniforms[key] = value2[0:2] elif len(value2) == 2: uniforms[key] = value2 else: raise AssertionError("precondition violation, optunity searchspace needs list with left and right range bounds!") if len(cat) == 0: return uniforms # build nested categorical structure inner_level = uniforms for key, value in cat.items(): tmp = {} tmp2 = {} for key2, value2 in value.items(): if key2 == 'data': for elem in value2: tmp[elem] = inner_level tmp2[key] = tmp inner_level = tmp2 solution_space = tmp2 return solution_space diff --git a/hyppopy/Solver/RandomsearchSolver.py b/hyppopy/Solver/RandomsearchSolver.py index aa297c3..4938fa5 100644 --- a/hyppopy/Solver/RandomsearchSolver.py +++ b/hyppopy/Solver/RandomsearchSolver.py @@ -1,159 +1,157 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE -# -# Author: Sven Wanner (s.wanner@dkfz.de) import os import copy import random import logging import numpy as np from pprint import pformat from hyppopy.globals import DEBUGLEVEL from hyppopy.solver.HyppopySolver import HyppopySolver LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) def draw_uniform_sample(param): """ function draws a random sample from a uniform range :param param: [dict] input hyperparameter discription :return: random sample value of type data['type'] """ assert param['type'] != 'str', "cannot sample a string list!" assert param['data'][0] < param['data'][1], "precondition violation: data[0] > data[1]!" s = random.random() s *= np.abs(param['data'][1] - param['data'][0]) s += param['data'][0] if param['type'] == 'int': s = int(np.round(s)) if s < param['data'][0]: s = int(param['data'][0]) if s > param['data'][1]: s = int(param['data'][1]) return s def draw_normal_sample(param): """ function draws a random sample from a normal distributed range :param param: [dict] input hyperparameter discription :return: random sample value of type data['type'] """ assert param['type'] != 'str', "cannot sample a string list!" assert param['data'][0] < param['data'][1], "precondition violation: data[0] > data[1]!" mu = (param['data'][1] - param['data'][0]) / 2 sigma = mu / 3 s = np.random.normal(loc=param['data'][0] + mu, scale=sigma) if s > param['data'][1]: s = param['data'][1] if s < param['data'][0]: s = param['data'][0] s = float(s) if param["type"] == "int": s = int(np.round(s)) return s def draw_loguniform_sample(param): """ function draws a random sample from a logarithmic distributed range :param param: [dict] input hyperparameter discription :return: random sample value of type data['type'] """ assert param['type'] != 'str', "cannot sample a string list!" assert param['data'][0] < param['data'][1], "precondition violation: data[0] > data[1]!" p = copy.deepcopy(param) p['data'][0] = np.log(param['data'][0]) p['data'][1] = np.log(param['data'][1]) assert p['data'][0] is not np.nan, "Precondition violation, left bound input error, results in nan!" assert p['data'][1] is not np.nan, "Precondition violation, right bound input error, results in nan!" x = draw_uniform_sample(p) s = np.exp(x) if s > param['data'][1]: s = param['data'][1] if s < param['data'][0]: s = param['data'][0] return s def draw_categorical_sample(param): """ function draws a random sample from a categorical list :param param: [dict] input hyperparameter discription :return: random sample value of type data['type'] """ return random.sample(param['data'], 1)[0] def draw_sample(param): """ function draws a sample from the input hyperparameter descriptor depending on it's domain :param param: [dict] input hyperparameter discription :return: random sample value of type data['type'] """ assert isinstance(param, dict), "input error, hyperparam descriptors of type {} not allowed!".format(type(param)) assert 'domain' in param.keys(), "input error, hyperparam descriptors need a domain key!" assert 'data' in param.keys(), "input error, hyperparam descriptors need a data key!" assert 'type' in param.keys(), "input error, hyperparam descriptors need a type key!" if param['domain'] == "uniform": return draw_uniform_sample(param) elif param['domain'] == "normal": return draw_normal_sample(param) elif param['domain'] == "loguniform": return draw_loguniform_sample(param) elif param['domain'] == "categorical": return draw_categorical_sample(param) else: raise LookupError("Unknown domain {}".format(param['domain'])) class RandomsearchSolver(HyppopySolver): """ The RandomsearchSolver class implements a randomsearch optimization. The randomsearch supports categorical, uniform, normal and loguniform sampling. The solver draws an independent sample from the parameter space each iteration.""" def __init__(self, project=None): HyppopySolver.__init__(self, project) def loss_function_call(self, params): loss = self.blackbox(**params) if loss is None: return np.nan return loss def execute_solver(self, searchspace): N = self.max_iterations try: for n in range(N): params = {} for name, p in searchspace.items(): params[name] = draw_sample(p) self.loss_function(**params) except Exception as e: msg = "internal error in randomsearch execute_solver occured. {}".format(e) LOG.error(msg) raise BrokenPipeError(msg) self.best = self._trials.argmin def convert_searchspace(self, hyperparameter): """ this function simply pipes the input parameter through, the sample drawing functions are responsible for interpreting the parameter. :param hyperparameter: [dict] hyperparameter space :return: [dict] hyperparameter space """ LOG.debug("convert input parameter\n\n\t{}\n".format(pformat(hyperparameter))) return hyperparameter diff --git a/hyppopy/SolverPool.py b/hyppopy/SolverPool.py index 9eb406f..205eed3 100644 --- a/hyppopy/SolverPool.py +++ b/hyppopy/SolverPool.py @@ -1,81 +1,79 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE -# -# Author: Sven Wanner (s.wanner@dkfz.de) from .Singleton import * import os import logging from hyppopy.HyppopyProject import HyppopyProject from hyppopy.solver.OptunaSolver import OptunaSolver from hyppopy.solver.BayesOptSolver import BayesOptSolver from hyppopy.solver.HyperoptSolver import HyperoptSolver from hyppopy.solver.OptunitySolver import OptunitySolver from hyppopy.solver.GridsearchSolver import GridsearchSolver from hyppopy.solver.RandomsearchSolver import RandomsearchSolver from hyppopy.globals import DEBUGLEVEL LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) @singleton_object class SolverPool(metaclass=Singleton): def __init__(self): self._solver_list = ["hyperopt", "optunity", "bayesopt", "optuna", "randomsearch", "gridsearch"] def get_solver_names(self): return self._solver_list def get(self, solver_name=None, project=None): if solver_name is not None: assert isinstance(solver_name, str), "precondition violation, solver_name type str expected, got {} instead!".format(type(solver_name)) if project is not None: assert isinstance(project, HyppopyProject), "precondition violation, project type HyppopyProject expected, got {} instead!".format(type(project)) if "custom_use_solver" in project.__dict__: solver_name = project.custom_use_solver if solver_name not in self._solver_list: raise AssertionError("Solver named [{}] not implemented!".format(solver_name)) if solver_name == "hyperopt": if project is not None: return HyperoptSolver(project) return HyperoptSolver() elif solver_name == "optunity": if project is not None: return OptunitySolver(project) return OptunitySolver() elif solver_name == "bayesopt": if project is not None: return BayesOptSolver(project) return BayesOptSolver() elif solver_name == "optuna": if project is not None: return OptunaSolver(project) return OptunaSolver() elif solver_name == "gridsearch": if project is not None: return GridsearchSolver(project) return GridsearchSolver() elif solver_name == "randomsearch": if project is not None: return RandomsearchSolver(project) return RandomsearchSolver() diff --git a/hyppopy/VirtualFunction.py b/hyppopy/VirtualFunction.py index 67950bc..b7af171 100644 --- a/hyppopy/VirtualFunction.py +++ b/hyppopy/VirtualFunction.py @@ -1,225 +1,223 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE -# -# Author: Sven Wanner (s.wanner@dkfz.de) ######################################################################################################################## # USAGE # # The class VirtualFunction is meant to be a virtual energy function with an arbitrary dimensionality. The user can # simply scribble functions as a binary image using e.g. Gimp, defining their ranges using .cfg file and loading them # into the VirtualFunction. An instance of the class can then be used like a normal function returning the sampling of # each dimension loaded. # # 1. create binary images (IMPORTANT same shape for each), background black the function signature white, ensure that # each column has a white pixel. If more than one pixel appears in a column, only the lowest will be used. # # 2. create a .cfg file, see an example in hyppopy/virtualparameterspace # # 3. vfunc = VirtualFunction() # vfunc.load_images(path/of/your/binaryfiles/and/the/configfile) # # 4. use vfunc like a normal function, if you loaded 4 dimension binary images use it like f = vfunc(a,b,c,d) ######################################################################################################################## import os import sys import numpy as np import configparser from glob import glob import matplotlib.pyplot as plt import matplotlib.image as mpimg from hyppopy.globals import VFUNCDATAPATH class VirtualFunction(object): def __init__(self): self.config = None self.data = None self.axis = [] def __call__(self, *args, **kwargs): if len(kwargs) == self.dims(): args = [0]*len(kwargs) for key, value in kwargs.items(): index = int(key.split("_")[1]) args[index] = value assert len(args) == self.dims(), "wrong number of arguments!" for i in range(len(args)): assert self.axis[i][0] <= args[i] <= self.axis[i][1], "out of range access on axis {}!".format(i) lpos, rpos, fracs = self.pos_to_indices(args) fl = self.data[(list(range(self.dims())), lpos)] fr = self.data[(list(range(self.dims())), rpos)] return np.sum(fl*np.array(fracs) + fr*(1-np.array(fracs))) def clear(self): self.axis.clear() self.data = None self.config = None def dims(self): return self.data.shape[0] def size(self): return self.data.shape[1] def range(self, dim): return np.abs(self.axis[dim][1] - self.axis[dim][0]) def minima(self): glob_mins = [] for dim in range(self.dims()): x = [] fmin = np.min(self.data[dim, :]) for _x in range(self.size()): if self.data[dim, _x] <= fmin: x.append(_x/self.size()*(self.axis[dim][1]-self.axis[dim][0])+self.axis[dim][0]) glob_mins.append([x, fmin]) return glob_mins def pos_to_indices(self, positions): lpos = [] rpos = [] pfracs = [] for n in range(self.dims()): pos = positions[n] pos -= self.axis[n][0] pos /= np.abs(self.axis[n][1]-self.axis[n][0]) pos *= self.data.shape[1]-1 lp = int(np.floor(pos)) if lp < 0: lp = 0 rp = int(np.ceil(pos)) if rp > self.data.shape[1]-1: rp = self.data.shape[1]-1 pfracs.append(1.0-(pos-np.floor(pos))) lpos.append(lp) rpos.append(rp) return lpos, rpos, pfracs def plot(self, dim=None, title=""): if dim is None: dim = list(range(self.dims())) else: dim = [dim] fig = plt.figure(figsize=(10, 8)) for i in range(len(dim)): width = np.abs(self.axis[dim[i]][1]-self.axis[dim[i]][0]) ax = np.arange(self.axis[dim[i]][0], self.axis[dim[i]][1], width/self.size()) plt.plot(ax, self.data[dim[i], :], '.', label='axis_{}'.format(str(dim[i]).zfill(2))) plt.legend() plt.grid() plt.title(title) plt.show() def add_dimension(self, data, x_range): if self.data is None: self.data = data if len(self.data.shape) == 1: self.data = self.data.reshape((1, self.data.shape[0])) else: if len(data.shape) == 1: data = data.reshape((1, data.shape[0])) assert self.data.shape[1] == data.shape[1], "shape mismatch while adding dimension!" dims = self.data.shape[0] size = self.data.shape[1] tmp = np.append(self.data, data) self.data = tmp.reshape((dims+1, size)) self.axis.append(x_range) def load_default(self, name="3D"): path = os.path.join(VFUNCDATAPATH, "{}".format(name)) if os.path.exists(path): self.load_images(path) else: raise FileExistsError("No virtualfunction of dimension {} available".format(name)) def load_images(self, path): self.config = None self.data = None self.axis.clear() img_fnames = [] for f in glob(path + os.sep + "*"): if f.endswith(".png"): img_fnames.append(f) elif f.endswith(".cfg"): self.config = self.read_config(f) else: print("WARNING: files of type {} not supported, the file {} is ignored!".format(f.split(".")[-1], os.path.basename(f))) if self.config is None: print("Aborted, failed to read configfile!") sys.exit() sections = self.config.sections() if len(sections) != len(img_fnames): print("Aborted, inconsistent number of image tmplates and axis specifications!") sys.exit() img_fnames.sort() size_x = None size_y = None for n, fname in enumerate(img_fnames): img = mpimg.imread(fname) if len(img.shape) > 2: img = img[:, :, 0] if size_x is None: size_x = img.shape[1] if size_y is None: size_y = img.shape[0] self.data = np.zeros((len(img_fnames), size_x), dtype=np.float32) assert img.shape[0] == size_y, "Shape mismatch in dimension y {} is not {}".format(img.shape[0], size_y) assert img.shape[1] == size_x, "Shape mismatch in dimension x {} is not {}".format(img.shape[1], size_x) self.sample_image(img, n) def sample_image(self, img, dim): sec_name = "axis_{}".format(str(dim).zfill(2)) assert sec_name in self.config.sections(), "config section {} not found!".format(sec_name) settings = self.get_axis_settings(sec_name) self.axis.append([float(settings['min_x']), float(settings['max_x'])]) y_range = [float(settings['min_y']), float(settings['max_y'])] for x in range(img.shape[1]): candidates = np.where(img[:, x] > 0) assert len(candidates[0]) > 0, "non function value in image detected, ensure each column has at least one value > 0!" y_pos = candidates[0][0]/img.shape[0] self.data[dim, x] = 1-y_pos self.data[dim, :] *= np.abs(y_range[1] - y_range[0]) self.data[dim, :] += y_range[0] def read_config(self, fname): try: config = configparser.ConfigParser() config.read(fname) return config except Exception as e: print(e) return None def get_axis_settings(self, section): dict1 = {} options = self.config.options(section) for option in options: try: dict1[option] = self.config.get(section, option) if dict1[option] == -1: print("skip: %s" % option) except: print("exception on %s!" % option) dict1[option] = None return dict1 diff --git a/hyppopy/VisdomViewer.py b/hyppopy/VisdomViewer.py new file mode 100644 index 0000000..bff9a94 --- /dev/null +++ b/hyppopy/VisdomViewer.py @@ -0,0 +1,114 @@ +import warnings +import numpy as np +from visdom import Visdom +import matplotlib.pyplot as plt + + +def time_formatter(time_s): + if time_s < 0.01: + return int(time_s * 1000.0 * 1000) / 1000.0, "ms" + elif 100 < time_s < 3600: + return int(time_s / 60 * 1000) / 1000.0, "min" + elif time_s >= 3600: + return int(time_s / 3600 * 1000) / 1000.0, "h" + else: + return int(time_s * 1000) / 1000.0, "s" + + +class VisdomViewer(object): + + def __init__(self, project, port=8097, server="http://localhost"): + self._viz = Visdom(port=port, server=server) + self._enabled = self._viz.check_connection(timeout_seconds=3) + if not self._enabled: + warnings.warn("No connection to visdom server established. Visualization cannot be displayed!") + + self._project = project + self._best_win = None + self._best_loss = None + self._loss_iter_plot = None + self._status_report = None + self._axis_tags = None + self._axis_plots = None + + def plot_losshistory(self, input_data): + loss = np.array([input_data["loss"]]) + iter = np.array([input_data["iterations"]]) + if self._loss_iter_plot is None: + self._loss_iter_plot = self._viz.line(loss, X=iter, opts=dict( + markers=True, + markersize=5, + dash=np.array(['dashdot']), + title="Loss History", + xlabel='iteration', + ylabel='loss' + )) + else: + self._viz.line(loss, X=iter, win=self._loss_iter_plot, update='append') + + def plot_hyperparameter(self, input_data): + if self._axis_plots is None: + self._axis_tags = [] + self._axis_plots = {} + for item in input_data.keys(): + if item == "refresh_time" or item == "book_time" or item == "iterations" or item == "status" or item == "loss": + continue + self._axis_tags.append(item) + for axis in self._axis_tags: + xlabel = "value" + if isinstance(input_data[axis], str): + if self._project.hyperparameter[axis]["domain"] == "categorical": + xlabel = '-'.join(self._project.hyperparameter[axis]["data"]) + input_data[axis] = self._project.hyperparameter[axis]["data"].index(input_data[axis]) + axis_loss = np.array([input_data[axis], input_data["loss"]]).reshape(1, -1) + self._axis_plots[axis] = self._viz.scatter(axis_loss, opts=dict( + markersize=5, + title=axis, + xlabel=xlabel, + ylabel='loss')) + else: + for axis in self._axis_tags: + if isinstance(input_data[axis], str): + if self._project.hyperparameter[axis]["domain"] == "categorical": + input_data[axis] = self._project.hyperparameter[axis]["data"].index(input_data[axis]) + axis_loss = np.array([input_data[axis], input_data["loss"]]).reshape(1, -1) + self._viz.scatter(axis_loss, win=self._axis_plots[axis], update='append') + + def show_statusreport(self, input_data): + duration = input_data['refresh_time'] - input_data['book_time'] + duration, time_format = time_formatter(duration.total_seconds()) + report = "Iteration {}: {}{} -> {}\n".format(input_data["iterations"], duration, time_format, input_data["status"]) + if self._status_report is None: + self._status_report = self._viz.text(report) + else: + self._viz.text(report, win=self._status_report, append=True) + + def show_best(self, input_data): + if self._best_win is None: + self._best_loss = input_data["loss"] + txt = "Best Parameter Set:
Loss: {}
" + self._best_win = self._viz.text(txt) + else: + if input_data["loss"] <= self._best_loss: + self._best_loss = input_data["loss"] + txt = "Best Parameter Set:
Loss: {}
" + self._viz.text(txt, win=self._best_win, append=False) + + def update(self, input_data): + if self._enabled: + self.show_statusreport(input_data) + self.plot_losshistory(input_data) + self.plot_hyperparameter(input_data) + self.show_best(input_data) diff --git a/hyppopy/__init__.py b/hyppopy/__init__.py index 6d7ce6a..cc999aa 100644 --- a/hyppopy/__init__.py +++ b/hyppopy/__init__.py @@ -1,16 +1,14 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE -# -# Author: Sven Wanner (s.wanner@dkfz.de) __version__ = '0.4.0.0' diff --git a/hyppopy/globals.py b/hyppopy/globals.py index f7347ae..4e701b0 100644 --- a/hyppopy/globals.py +++ b/hyppopy/globals.py @@ -1,37 +1,35 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE -# -# Author: Sven Wanner (s.wanner@dkfz.de) import os import sys import logging ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) sys.path.insert(0, ROOT) LIBNAME = "hyppopy" TESTDATA_DIR = os.path.join(ROOT, *(LIBNAME, "tests", "data")) HYPERPARAMETERPATH = "hyperparameter" SETTINGSPATH = "settings" VFUNCDATAPATH = os.path.join(os.path.join(ROOT, LIBNAME), "virtualparameterspace") SUPPORTED_DOMAINS = ["uniform", "normal", "loguniform", "categorical"] SUPPORTED_DTYPES = ["int", "float", "str"] DEFAULTITERATIONS = 500 LOGFILENAME = os.path.join(ROOT, '{}_log.log'.format(LIBNAME)) DEBUGLEVEL = logging.DEBUG logging.basicConfig(filename=LOGFILENAME, filemode='w', format='%(levelname)s: %(name)s - %(message)s') diff --git a/hyppopy/tests/test_bayesoptsolver.py b/hyppopy/tests/test_bayesoptsolver.py index 4bae53d..1847a0e 100644 --- a/hyppopy/tests/test_bayesoptsolver.py +++ b/hyppopy/tests/test_bayesoptsolver.py @@ -1,71 +1,69 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE -# -# Author: Sven Wanner (s.wanner@dkfz.de) import unittest import matplotlib.pylab as plt from hyppopy.solver.BayesOptSolver import * from hyppopy.VirtualFunction import VirtualFunction from hyppopy.HyppopyProject import HyppopyProject class BayesOptSolverTestSuite(unittest.TestCase): def setUp(self): pass def test_solver_complete(self): config = { "hyperparameter": { "axis_00": { "domain": "normal", "data": [300, 800], "type": "float" }, "axis_01": { "domain": "uniform", "data": [-1, 1], "type": "float" }, "axis_02": { "domain": "uniform", "data": [0, 10], "type": "float" } }, "settings": { "solver": {"max_iterations": 10}, "custom": {} }} project = HyppopyProject(config) solver = BayesOptSolver(project) vfunc = VirtualFunction() vfunc.load_default() solver.blackbox = vfunc solver.run(print_stats=False) df, best = solver.get_results() self.assertTrue(300 <= best['axis_00'] <= 800) self.assertTrue(-1 <= best['axis_01'] <= 1) self.assertTrue(0 <= best['axis_02'] <= 10) for status in df['status']: self.assertTrue(status) for loss in df['losses']: self.assertTrue(isinstance(loss, float)) if __name__ == '__main__': unittest.main() diff --git a/hyppopy/tests/test_gridsearchsolver.py b/hyppopy/tests/test_gridsearchsolver.py index 57b6b19..07ee364 100644 --- a/hyppopy/tests/test_gridsearchsolver.py +++ b/hyppopy/tests/test_gridsearchsolver.py @@ -1,211 +1,209 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE -# -# Author: Sven Wanner (s.wanner@dkfz.de) import unittest from hyppopy.solver.GridsearchSolver import * from hyppopy.VirtualFunction import VirtualFunction from hyppopy.HyppopyProject import HyppopyProject class GridsearchTestSuite(unittest.TestCase): def setUp(self): pass def test_get_uniform_axis_sample(self): drange = [0, 10] N = 11 data = get_uniform_axis_sample(drange[0], drange[1], N, "float") for i in range(11): self.assertEqual(float(i), data[i]) drange = [-10, 10] N = 21 data = get_uniform_axis_sample(drange[0], drange[1], N, "int") self.assertEqual(data[0], -10) self.assertEqual(data[20], 10) self.assertEqual(data[10], 0) def test_get_norm_cdf(self): res = [0, 0.27337265, 0.4331928, 0.48777553, 0.4986501, 0.5013499, 0.51222447, 0.5668072, 0.72662735, 1] f = get_norm_cdf(10) for n, v in enumerate(res): self.assertAlmostEqual(v, f[n]) res = [0.0, 0.27337264762313174, 0.4331927987311419, 0.48777552734495533, 0.4986501019683699, 0.5, 0.5013498980316301, 0.5122244726550447, 0.5668072012688581, 0.7266273523768683, 1.0] f = get_norm_cdf(11) for n, v in enumerate(res): self.assertAlmostEqual(v, f[n]) def test_get_gaussian_axis_sampling(self): res = [-5.0, -2.2662735237686826, -0.6680720126885813, -0.12224472655044671, -0.013498980316301257, 0.013498980316301257, 0.12224472655044671, 0.6680720126885813, 2.2662735237686826, 5.0] bounds = (-5, 5) N = 10 data = get_gaussian_axis_sample(bounds[0], bounds[1], N, "float") for n in range(N): self.assertAlmostEqual(res[n], data[n]) res = [-5.0, -2.2662735237686826, -0.6680720126885813, -0.12224472655044671, -0.013498980316301257, 0.0, 0.013498980316301257, 0.12224472655044671, 0.6680720126885813, 2.2662735237686826, 5.0] bounds = (-5, 5) N = 11 data = get_gaussian_axis_sample(bounds[0], bounds[1], N, "float") for n in range(N): self.assertAlmostEqual(res[n], data[n]) def test_get_logarithmic_axis_sample(self): res = [0.0010000000000000002, 0.0035938136638046297, 0.012915496650148841, 0.046415888336127795, 0.1668100537200059, 0.5994842503189414, 2.154434690031884, 7.7426368268112675, 27.825594022071247, 100.00000000000004] bounds = (0.001, 1e2) N = 10 data = get_logarithmic_axis_sample(bounds[0], bounds[1], N, "float") for n in range(N): self.assertAlmostEqual(res[n], data[n]) res = [0.0010000000000000002, 0.003162277660168382, 0.010000000000000004, 0.03162277660168381, 0.10000000000000006, 0.31622776601683833, 1.0000000000000009, 3.1622776601683813, 10.00000000000001, 31.622776601683846, 100.00000000000004] bounds = (0.001, 1e2) N = 11 data = get_logarithmic_axis_sample(bounds[0], bounds[1], N, "float") for n in range(N): self.assertAlmostEqual(res[n], data[n]) def test_solver(self): config = { "hyperparameter": { "value 1": { "domain": "uniform", "data": [0, 20, 11], "type": "int" }, "value 2": { "domain": "normal", "data": [0, 20.0, 11], "type": "float" }, "value 3": { "domain": "loguniform", "data": [1, 10000, 11], "type": "float" }, "categorical": { "domain": "categorical", "data": ["a", "b"], "type": "str" } }, "settings": { "solver": {}, "custom": {} }} res_labels = ['value 1', 'value 2', 'value 3', 'categorical'] res_values = [[0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20], [0.0, 5.467452952462635, 8.663855974622837, 9.755510546899107, 9.973002039367397, 10.0, 10.026997960632603, 10.244489453100893, 11.336144025377163, 14.532547047537365, 20.0], [1.0, 2.51188643150958, 6.309573444801933, 15.848931924611136, 39.810717055349734, 100.00000000000004, 251.18864315095806, 630.9573444801938, 1584.8931924611143, 3981.071705534977, 10000.00000000001], ['a', 'b'] ] solver = GridsearchSolver(config) searchspace = solver.convert_searchspace(config["hyperparameter"]) for n in range(len(res_labels)): self.assertEqual(res_labels[n], searchspace[0][n]) for i in range(3): self.assertAlmostEqual(res_values[i], searchspace[1][i]) self.assertEqual(res_values[3], searchspace[1][3]) def test_solver_complete(self): config = { "hyperparameter": { "axis_00": { "domain": "normal", "data": [300, 800, 11], "type": "float" }, "axis_01": { "domain": "normal", "data": [-1, 1, 11], "type": "float" }, "axis_02": { "domain": "uniform", "data": [0, 10, 11], "type": "float" } }, "settings": { "solver": {}, "custom": {} }} project = HyppopyProject(config) solver = GridsearchSolver(project) vfunc = VirtualFunction() vfunc.load_default() solver.blackbox = vfunc solver.run(print_stats=False) df, best = solver.get_results() self.assertAlmostEqual(best['axis_00'], 583.40, places=1) self.assertAlmostEqual(best['axis_01'], 0.45, places=1) self.assertAlmostEqual(best['axis_02'], 5.0, places=1) for status in df['status']: self.assertTrue(status) for loss in df['losses']: self.assertTrue(isinstance(loss, float)) if __name__ == '__main__': unittest.main() diff --git a/hyppopy/tests/test_hyperoptsolver.py b/hyppopy/tests/test_hyperoptsolver.py index a2be31c..02c043d 100644 --- a/hyppopy/tests/test_hyperoptsolver.py +++ b/hyppopy/tests/test_hyperoptsolver.py @@ -1,71 +1,69 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE -# -# Author: Sven Wanner (s.wanner@dkfz.de) import unittest import matplotlib.pylab as plt from hyppopy.solver.HyperoptSolver import * from hyppopy.VirtualFunction import VirtualFunction from hyppopy.HyppopyProject import HyppopyProject class HyperoptSolverTestSuite(unittest.TestCase): def setUp(self): pass def test_solver_complete(self): config = { "hyperparameter": { "axis_00": { "domain": "normal", "data": [300, 800], "type": "float" }, "axis_01": { "domain": "normal", "data": [-1, 1], "type": "float" }, "axis_02": { "domain": "uniform", "data": [0, 10], "type": "float" } }, "settings": { "solver": {"max_iterations": 100}, "custom": {} }} project = HyppopyProject(config) solver = HyperoptSolver(project) vfunc = VirtualFunction() vfunc.load_default() solver.blackbox = vfunc solver.run(print_stats=False) df, best = solver.get_results() self.assertTrue(300 <= best['axis_00'] <= 800) self.assertTrue(-1 <= best['axis_01'] <= 1) self.assertTrue(0 <= best['axis_02'] <= 10) for status in df['status']: self.assertTrue(status) for loss in df['losses']: self.assertTrue(isinstance(loss, float)) if __name__ == '__main__': unittest.main() diff --git a/hyppopy/tests/test_hyppopyproject.py b/hyppopy/tests/test_hyppopyproject.py index a093ce3..5f9e347 100644 --- a/hyppopy/tests/test_hyppopyproject.py +++ b/hyppopy/tests/test_hyppopyproject.py @@ -1,97 +1,95 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE -# -# Author: Sven Wanner (s.wanner@dkfz.de) import os import unittest import numpy as np from hyppopy.HyppopyProject import HyppopyProject from hyppopy.globals import TESTDATA_DIR def foo(a, b): return a + b class VirtualFunctionTestSuite(unittest.TestCase): def setUp(self): pass def test_project_creation(self): config = { "hyperparameter": { "C": { "domain": "uniform", "data": [0.0001, 20], "type": "float" }, "kernel": { "domain": "categorical", "data": ["linear", "sigmoid", "poly", "rbf"], "type": "str" } }, "settings": { "solver": { "max_iterations": 300 }, "custom": { "param1": 1, "param2": 2, "function": foo } }} project = HyppopyProject() project.set_config(config) self.assertEqual(project.hyperparameter["C"]["domain"], "uniform") self.assertEqual(project.hyperparameter["C"]["data"], [0.0001, 20]) self.assertEqual(project.hyperparameter["C"]["type"], "float") self.assertEqual(project.hyperparameter["kernel"]["domain"], "categorical") self.assertEqual(project.hyperparameter["kernel"]["data"], ["linear", "sigmoid", "poly", "rbf"]) self.assertEqual(project.hyperparameter["kernel"]["type"], "str") self.assertEqual(project.solver_max_iterations, 300) self.assertEqual(project.custom_param1, 1) self.assertEqual(project.custom_param2, 2) self.assertEqual(project.custom_function(2, 3), 5) self.assertTrue(project.get_typeof("C") is float) self.assertTrue(project.get_typeof("kernel") is str) project.clear() self.assertTrue(len(project.hyperparameter) == 0) self.assertTrue(len(project.settings) == 0) self.assertTrue("solver_max_iterations" not in project.__dict__.keys()) self.assertTrue("custom_param1" not in project.__dict__.keys()) self.assertTrue("custom_param2" not in project.__dict__.keys()) self.assertTrue("custom_function" not in project.__dict__.keys()) project.add_hyperparameter(name="C", domain="uniform", data=[0.0001, 20], dtype="float") project.add_hyperparameter(name="kernel", domain="categorical", data=["linear", "sigmoid", "poly", "rbf"], dtype="str") self.assertEqual(project.hyperparameter["C"]["domain"], "uniform") self.assertEqual(project.hyperparameter["C"]["data"], [0.0001, 20]) self.assertEqual(project.hyperparameter["C"]["type"], "float") self.assertEqual(project.hyperparameter["kernel"]["domain"], "categorical") self.assertEqual(project.hyperparameter["kernel"]["data"], ["linear", "sigmoid", "poly", "rbf"]) self.assertEqual(project.hyperparameter["kernel"]["type"], "str") project.add_settings("solver", "max_iterations", 500) self.assertEqual(project.solver_max_iterations, 500) project.add_settings("solver", "max_iterations", 200) self.assertEqual(project.solver_max_iterations, 200) diff --git a/hyppopy/tests/test_optunasolver.py b/hyppopy/tests/test_optunasolver.py index 523dba1..b9ec521 100644 --- a/hyppopy/tests/test_optunasolver.py +++ b/hyppopy/tests/test_optunasolver.py @@ -1,71 +1,69 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE -# -# Author: Sven Wanner (s.wanner@dkfz.de) import unittest import matplotlib.pylab as plt from hyppopy.solver.OptunaSolver import * from hyppopy.VirtualFunction import VirtualFunction from hyppopy.HyppopyProject import HyppopyProject class OptunaSolverTestSuite(unittest.TestCase): def setUp(self): pass def test_solver_complete(self): config = { "hyperparameter": { "axis_00": { "domain": "normal", "data": [300, 800], "type": "float" }, "axis_01": { "domain": "normal", "data": [-1, 1], "type": "float" }, "axis_02": { "domain": "uniform", "data": [0, 10], "type": "float" } }, "settings": { "solver": {"max_iterations": 100}, "custom": {} }} project = HyppopyProject(config) solver = OptunaSolver(project) vfunc = VirtualFunction() vfunc.load_default() solver.blackbox = vfunc solver.run(print_stats=False) df, best = solver.get_results() self.assertTrue(300 <= best['axis_00'] <= 800) self.assertTrue(-1 <= best['axis_01'] <= 1) self.assertTrue(0 <= best['axis_02'] <= 10) for status in df['status']: self.assertTrue(status) for loss in df['losses']: self.assertTrue(isinstance(loss, float)) if __name__ == '__main__': unittest.main() diff --git a/hyppopy/tests/test_optunitysolver.py b/hyppopy/tests/test_optunitysolver.py index 287df1e..f0a617a 100644 --- a/hyppopy/tests/test_optunitysolver.py +++ b/hyppopy/tests/test_optunitysolver.py @@ -1,71 +1,69 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE -# -# Author: Sven Wanner (s.wanner@dkfz.de) import unittest import matplotlib.pylab as plt from hyppopy.solver.OptunitySolver import * from hyppopy.VirtualFunction import VirtualFunction from hyppopy.HyppopyProject import HyppopyProject class OptunitySolverTestSuite(unittest.TestCase): def setUp(self): pass def test_solver_complete(self): config = { "hyperparameter": { "axis_00": { "domain": "normal", "data": [300, 800], "type": "float" }, "axis_01": { "domain": "normal", "data": [-1, 1], "type": "float" }, "axis_02": { "domain": "uniform", "data": [0, 10], "type": "float" } }, "settings": { "solver": {"max_iterations": 100}, "custom": {} }} project = HyppopyProject(config) solver = OptunitySolver(project) vfunc = VirtualFunction() vfunc.load_default() solver.blackbox = vfunc solver.run(print_stats=False) df, best = solver.get_results() self.assertTrue(300 <= best['axis_00'] <= 800) self.assertTrue(-1 <= best['axis_01'] <= 1) self.assertTrue(0 <= best['axis_02'] <= 10) for status in df['status']: self.assertTrue(status) for loss in df['losses']: self.assertTrue(isinstance(loss, float)) if __name__ == '__main__': unittest.main() diff --git a/hyppopy/tests/test_randomsearchsolver.py b/hyppopy/tests/test_randomsearchsolver.py index 0958798..772b1e9 100644 --- a/hyppopy/tests/test_randomsearchsolver.py +++ b/hyppopy/tests/test_randomsearchsolver.py @@ -1,137 +1,135 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE -# -# Author: Sven Wanner (s.wanner@dkfz.de) import unittest import matplotlib.pylab as plt from hyppopy.solver.RandomsearchSolver import * from hyppopy.VirtualFunction import VirtualFunction from hyppopy.HyppopyProject import HyppopyProject class RandomsearchTestSuite(unittest.TestCase): def setUp(self): pass def test_draw_uniform_sample(self): param = {"data": [0, 1, 10], "type": "float"} values = [] for i in range(10000): values.append(draw_uniform_sample(param)) self.assertTrue(0 <= values[-1] <= 1) self.assertTrue(isinstance(values[-1], float)) hist = plt.hist(values, bins=10, normed=True) std = np.std(hist[0]) mean = np.mean(hist[0]) self.assertTrue(std < 0.05) self.assertTrue(0.9 < mean < 1.1) param = {"data": [0, 10, 11], "type": "int"} values = [] for i in range(10000): values.append(draw_uniform_sample(param)) self.assertTrue(0 <= values[-1] <= 10) self.assertTrue(isinstance(values[-1], int)) hist = plt.hist(values, bins=11, normed=True) std = np.std(hist[0]) mean = np.mean(hist[0]) self.assertTrue(std < 0.05) self.assertTrue(0.09 < mean < 0.11) def test_draw_normal_sample(self): param = {"data": [0, 10, 11], "type": "int"} values = [] for i in range(10000): values.append(draw_normal_sample(param)) self.assertTrue(0 <= values[-1] <= 10) self.assertTrue(isinstance(values[-1], int)) hist = plt.hist(values, bins=11, normed=True) for i in range(1, 5): self.assertTrue(hist[0][i-1]-hist[0][i] < 0) for i in range(5, 10): self.assertTrue(hist[0][i] - hist[0][i+1] > 0) def test_draw_loguniform_sample(self): param = {"data": [1, 1000, 11], "type": "float"} values = [] for i in range(10000): values.append(draw_loguniform_sample(param)) self.assertTrue(1 <= values[-1] <= 1000) self.assertTrue(isinstance(values[-1], float)) hist = plt.hist(values, bins=11, normed=True) for i in range(4): self.assertTrue(hist[0][i] > hist[0][i+1]) self.assertTrue((hist[0][i] - hist[0][i+1]) > 0) def test_draw_categorical_sample(self): param = {"data": [1, 2, 3], "type": int} values = [] for i in range(10000): values.append(draw_categorical_sample(param)) self.assertTrue(values[-1] == 1 or values[-1] == 2 or values[-1] == 3) self.assertTrue(isinstance(values[-1], int)) hist = plt.hist(values, bins=3, normed=True) for i in range(3): self.assertTrue(0.45 < hist[0][i] < 0.55) def test_solver_complete(self): config = { "hyperparameter": { "axis_00": { "domain": "normal", "data": [300, 800], "type": "float" }, "axis_01": { "domain": "uniform", "data": [-1, 1], "type": "float" }, "axis_02": { "domain": "uniform", "data": [0, 10], "type": "float" } }, "settings": { "solver": {"max_iterations": 100}, "custom": {} }} project = HyppopyProject(config) solver = RandomsearchSolver(project) vfunc = VirtualFunction() vfunc.load_default() solver.blackbox = vfunc solver.run(print_stats=False) df, best = solver.get_results() self.assertTrue(300 <= best['axis_00'] <= 800) self.assertTrue(-1 <= best['axis_01'] <= 1) self.assertTrue(0 <= best['axis_02'] <= 10) for status in df['status']: self.assertTrue(status) for loss in df['losses']: self.assertTrue(isinstance(loss, float)) if __name__ == '__main__': unittest.main() diff --git a/hyppopy/tests/test_virtualfunction.py b/hyppopy/tests/test_virtualfunction.py index f807da0..9481111 100644 --- a/hyppopy/tests/test_virtualfunction.py +++ b/hyppopy/tests/test_virtualfunction.py @@ -1,96 +1,94 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE -# -# Author: Sven Wanner (s.wanner@dkfz.de) import os import unittest import numpy as np from hyppopy.VirtualFunction import VirtualFunction from hyppopy.globals import TESTDATA_DIR class VirtualFunctionTestSuite(unittest.TestCase): def setUp(self): pass def test_imagereading(self): vfunc = VirtualFunction() vfunc.load_images(os.path.join(TESTDATA_DIR, 'functionsimulator')) self.assertTrue(isinstance(vfunc.data, np.ndarray)) self.assertEqual(vfunc.data.shape[0], 5) self.assertEqual(vfunc.data.shape[1], 512) gt = [0.83984375*5, 0.44140625*20-10, 0.25390625*20, 0.81640625*8-10, 0.67578125*2+2] for i in range(5): self.assertAlmostEqual(vfunc.data[i][0], gt[i]) gt = [[0, 1], [-10, 10], [0, 20], [-30, 5], [5, 10]] for i in range(5): self.assertEqual(vfunc.axis[i][0], gt[i][0]) self.assertEqual(vfunc.axis[i][1], gt[i][1]) def test_data_adding(self): gt = [[-10, 10], [-30, 5]] vfunc = VirtualFunction() dim0 = np.arange(0, 1.1, 0.1) dim1 = np.arange(1.0, -0.1, -0.1) vfunc.add_dimension(dim0, gt[0]) self.assertEqual(len(vfunc.data.shape), 2) self.assertEqual(vfunc.data.shape[0], 1) self.assertEqual(vfunc.data.shape[1], 11) vfunc.add_dimension(dim1, gt[1]) self.assertEqual(vfunc.data.shape[0], 2) self.assertEqual(vfunc.data.shape[1], 11) for n in range(11): self.assertAlmostEqual(dim0[n], vfunc.data[0, n]) self.assertAlmostEqual(dim1[n], vfunc.data[1, n]) for i in range(2): self.assertEqual(vfunc.axis[i][0], gt[i][0]) self.assertEqual(vfunc.axis[i][1], gt[i][1]) def test_minima(self): vfunc = VirtualFunction() vfunc.load_images(os.path.join(TESTDATA_DIR, 'functionsimulator')) minima = vfunc.minima() gt = [[[0.7265625], 0.48828125], [[-4.0234375], -7.890625], [[2.265625], 0.859375], [ [-17.421875, -17.353515625, -17.28515625, -17.216796875, -17.1484375, -17.080078125, -17.01171875, -16.943359375, -16.875, -16.806640625, -16.73828125, -16.669921875, -16.6015625, -16.533203125, -16.46484375, -16.396484375, -16.328125, -16.259765625, -16.19140625, -16.123046875, -16.0546875, -15.986328125, -15.91796875, -15.849609375, -15.78125, -15.712890625, -15.64453125, -15.576171875, -15.5078125, -15.439453125, -15.37109375, -15.302734375, -15.234375, -15.166015625, -15.09765625, -15.029296875, -14.9609375, -14.892578125, -14.82421875, -14.755859375, -14.6875, -14.619140625, -14.55078125, -14.482421875, -14.4140625, -14.345703125, -14.27734375, -14.208984375, -14.140625, -14.072265625, -14.00390625, -13.935546875, -13.8671875, -13.798828125, -13.73046875, -13.662109375, -13.59375, -13.525390625, -13.45703125, -13.388671875, -13.3203125, -13.251953125, -13.18359375, -13.115234375, -13.046875, -12.978515625, -12.91015625, -12.841796875, -12.7734375, -12.705078125, -12.63671875, -12.568359375, -12.5, -12.431640625, -12.36328125, -12.294921875, -12.2265625, -12.158203125, -12.08984375, -12.021484375, -11.953125, -11.884765625, -11.81640625, -11.748046875, -11.6796875, -11.611328125, -11.54296875, -11.474609375, -11.40625, -11.337890625, -11.26953125, -11.201171875, -11.1328125, -11.064453125, -10.99609375, -10.927734375, -10.859375, -10.791015625, -10.72265625, -10.654296875, -10.5859375, -10.517578125, -10.44921875, -10.380859375, -10.3125, -10.244140625, -10.17578125, -10.107421875, -10.0390625, -9.970703125, -9.90234375, -9.833984375, -9.765625, -9.697265625, -9.62890625, -9.560546875, -9.4921875, -9.423828125, -9.35546875, -9.287109375, -9.21875, -9.150390625, -9.08203125, -9.013671875, -8.9453125, -8.876953125, -8.80859375, -8.740234375, -8.671875, -8.603515625, -8.53515625, -8.466796875, -8.3984375, -8.330078125, -8.26171875, -8.193359375, -8.125, -8.056640625, -7.98828125, -7.919921875, -7.8515625, -7.783203125, -7.71484375, -7.646484375, -7.578125, -7.509765625, -7.44140625, -7.373046875, -7.3046875, -7.236328125, -7.16796875, -7.099609375, -7.03125], -9.125], [[5.44921875, 5.458984375, 5.46875, 5.478515625, 5.48828125, 5.498046875, 5.5078125, 5.517578125, 5.52734375], 2.09375]] self.assertAlmostEqual(minima, gt) if __name__ == '__main__': unittest.main() diff --git a/setup.py b/setup.py index 0c8a3b9..41fcd66 100644 --- a/setup.py +++ b/setup.py @@ -1,54 +1,54 @@ import os from setuptools import setup, find_packages with open('README.md') as f: readme = f.read() with open('LICENSE') as f: license = f.read() -VERSION = "0.4.0.2" +VERSION = "0.4.1.0" ROOT = os.path.dirname(os.path.realpath(__file__)) new_init = [] with open(os.path.join(ROOT, *("hyppopy", "__init__.py")), "r") as infile: for line in infile: new_init.append(line) for n in range(len(new_init)): if new_init[n].startswith("__version__"): split = line.split("=") new_init[n] = "__version__ = '" + VERSION + "'\n" with open(os.path.join(ROOT, *("hyppopy", "__init__.py")), "w") as outfile: outfile.writelines(new_init) setup( name='hyppopy', version=VERSION, description='Hyper-Parameter Optimization Toolbox for Blackboxfunction Optimization', long_description=readme, # if you want, put your own name here # (this would likely result in people sending you emails) author='Sven Wanner', author_email='s.wanner@dkfz.de', url='', license=license, packages=find_packages(exclude=('tests', 'doc')), package_data={}, # the requirements to install this project. # Since this one is so simple this is empty. install_requires=[ 'bayesian-optimization>=1.0.1', 'hyperopt>=0.1.2', 'matplotlib>=3.0.3', 'numpy>=1.16.2', 'optuna>=0.9.0', 'Optunity>=1.1.1', 'pandas>=0.24.2', 'pytest>=4.3.1', 'scikit-learn>=0.20.3', 'scipy>=1.2.1' ], )