diff --git a/LICENSE b/LICENSE
index 366acc1..1422535 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,38 +1,38 @@
=======================================================================
-Copyright (c) 2003-2012 German Cancer Research Center,
+Copyright (c) 2019 German Cancer Research Center,
Division of Medical Image Computing
All rights reserved.
Redistribution and use in source and binary forms, with or
without modification, are permitted provided that the
following conditions are met:
* Redistributions of source code must retain the above
copyright notice, this list of conditions and the
following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the
following disclaimer in the documentation and/or other
materials provided with the distribution.
* Neither the name of the German Cancer Research Center,
nor the names of its contributors may be used to endorse
or promote products derived from this software without
specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
=======================================================================
diff --git a/README.md b/README.md
index 17aa081..b1339d8 100644
--- a/README.md
+++ b/README.md
@@ -1,353 +1,374 @@
![docs_title_logo](./resources/docs_title_logo.png)
# A Hyper-Parameter Optimization Toolbox
## What is Hyppopy?
Hyppopy is a python toolbox for blackbox optimization. It's purpose is to offer a unified and easy to use interface to a collection of solver libraries. Currently provided solvers are:
* [Hyperopt](http://hyperopt.github.io/hyperopt/)
* [Optunity](https://optunity.readthedocs.io/en/latest/user/index.html)
* [Optuna](https://optuna.org/)
* [BayesianOptimization](https://github.com/fmfn/BayesianOptimization)
* Randomsearch Solver
* Gridsearch Solver
## Installation
1. clone the [Hyppopy](http:\\github.com) project from Github
2. (create a virtual environment), open a console (with your activated virtual env) and go to the hyppopy root folder
3. ```$ pip install -r requirements.txt```
4. ```$ python setup.py install```
## How to use Hyppopy?
#### The HyppopyProject class
The HyppopyProject class takes care all settings necessary for the solver and your workflow. To setup a HyppopyProject instance we can use a nested dictionary or the classes memberfunctions respectively.
```python
# Import the HyppopyProject class
from hyppopy.HyppopyProject import HyppopyProject
# Create a nested dict with a section hyperparameter. We define a 2 dimensional
# hyperparameter space with a numerical dimension named myNumber of type float and
# a uniform sampling. The second dimension is a categorical parameter of type string.
config = {
"hyperparameter": {
"myNumber": {
"domain": "uniform",
"data": [0, 100],
"type": "float"
},
"myOption": {
"domain": "categorical",
"data": ["a", "b", "c"],
"type": "str"
}
}}
# Create a HyppopyProject instance and pass the config dict to
# the constructor. Alternatively one can use set_config method.
project = HyppopyProject(config=config)
# To demonstrate the second option we clear the project
project.clear()
# and add the parameter again using the member function add_hyperparameter
project.add_hyperparameter(name="myNumber", domain="uniform", data=[0, 100], dtype="float")
project.add_hyperparameter(name="myOption", domain="categorical", data=["a", "b", "c"], dtype="str")
```
```python
from hyppopy.HyppopyProject import HyppopyProject
# We might have seen a warning: 'UserWarning: config dict had no
# section settings/solver/max_iterations, set default value: 500'
# when executing the example above. This is due to the fact that
# most solvers need a value for a maximum number of iterations.
# To take care of solver settings (there might be more in the future)
# one can set a second section called settings. The settings section
# again is splitted into a subsection 'solver' and a subsection 'custom'.
# When adding max_iterations to the section settings/solver we can change
# the number of iterations the solver is doing. All solver except of the
# GridsearchSolver make use of the value max_iterations.
# The usage of the custom section is demonstrated later.
config = {
"hyperparameter": {
"myNumber": {
"domain": "uniform",
"data": [0, 100],
"type": "float"
},
"myOption": {
"domain": "categorical",
"data": ["a", "b", "c"],
"type": "str"
}
},
"settings": {
"solver": {
"max_iterations": 500
},
"custom": {}
}}
project = HyppopyProject(config=config)
```
The settings added are automatically converted to a class member with a prefix_ where prefix is the name of the subsection. One can make use of this feature to build custom workflows by adding params to the custom section. More interesting is this feature when developing your own solver.
```python
from hyppopy.HyppopyProject import HyppopyProject
# Creating a HyppopyProject instance
project = HyppopyProject()
project.add_hyperparameter(name="x", domain="uniform", data=[-10, 10], dtype="float")
project.add_hyperparameter(name="y", domain="uniform", data=[-10, 10], dtype="float")
project.add_settings(section="solver", name="max_iterations", value=300)
project.add_settings(section="custom", name="my_param1", value=True)
project.add_settings(section="custom", name="my_param2", value=42)
print("What is max_iterations value? {}".format(project.solver_max_iterations))
if project.custom_my_param1:
print("What is the answer? {}".format(project.custom_my_param2))
else:
print("What is the answer? x")
```
#### The HyppopySolver classes
Each solver is a child of the HyppopySolver class. This is only interesting if you're planning to write a new solver, we will discuss this in the section Solver Development. All solvers we can use to optimize our blackbox function are part of the module 'hyppopy.solver'. Below is a list of all solvers available along with their access key in squared brackets.
* HyperoptSolver [hyperopt]
* OptunitySolver [optunity]
* OptunaSolver [optuna]
* BayesOptSolver [bayesopt]
* RandomsearchSolver [randomsearch]
* GridsearchSolver [gridsearch]
There are two options to get a solver, we can import directly from the hyppopy.solver package or we use the SolverPool class. We look into both options by optimizing a simple function, starting with the direct import case.
```python
# Import the HyppopyProject class
from hyppopy.HyppopyProject import HyppopyProject
# Import the HyperoptSolver class, in this case wh use Hyperopt
from hyppopy.solver.HyperoptSolver import HyperoptSolver
# Our function to optimize
def my_loss_func(x, y):
return x**2+y**2
# Creating a HyppopyProject instance
project = HyppopyProject()
project.add_hyperparameter(name="x", domain="uniform", data=[-10, 10], dtype="float")
project.add_hyperparameter(name="y", domain="uniform", data=[-10, 10], dtype="float")
project.add_settings(section="solver", name="max_iterations", value=300)
# create a solver instance
solver = HyperoptSolver(project)
# pass the loss function to the solver
solver.blackbox = my_loss_func
# run the solver
solver.run()
df, best = solver.get_results()
print("\n")
print("*"*100)
print("Best Parameter Set:\n{}".format(best))
print("*"*100)
```
The SolverPool is a class keeping track of all solver classes. We have several options to ask the SolverPool for the desired solver. We can add an option called use_solver to our settings/custom section or to the project instance respectively, or we can use the solver access key (see solver listing above) to ask for the solver directly.
```python
# import the SolverPool class
from hyppopy.SolverPool import SolverPool
# Import the HyppopyProject class
from hyppopy.HyppopyProject import HyppopyProject
# Our function to optimize
def my_loss_func(x, y):
return x**2+y**2
# Creating a HyppopyProject instance
project = HyppopyProject()
project.add_hyperparameter(name="x", domain="uniform", data=[-10, 10], dtype="float")
project.add_hyperparameter(name="y", domain="uniform", data=[-10, 10], dtype="float")
project.add_settings(section="solver", name="max_iterations", value=300)
project.add_settings(section="custom", name="use_solver", value="hyperopt")
# create a solver instance. The SolverPool class is a singleton
# and can be used without instanciating. It looks in the project
# instance for the use_solver option and returns the correct solver.
solver = SolverPool.get(project=project)
# Another option without the usage of the use_solver field would be:
# solver = SolverPool.get(solver_name='hyperopt', project=project)
# pass the loss function to the solver
solver.blackbox = my_loss_func
# run the solver
solver.run()
df, best = solver.get_results()
print("\n")
print("*"*100)
print("Best Parameter Set:\n{}".format(best))
print("*"*100)
```
#### The BlackboxFunction class
To extend the possibilities beyond using parameter only loss function as in the examples above, the BlackboxFunction class can be used. This class is a wrapper class around the actual loss_function providing a more advanced access to data handling and a callback_function for accessing the solvers iteration loop.
```python
# import the HyppopyProject class keeping track of inputs
from hyppopy.HyppopyProject import HyppopyProject
# import the SolverPool singleton class
from hyppopy.SolverPool import SolverPool
# import the Blackboxfunction class wrapping your problem for Hyppopy
from hyppopy.BlackboxFunction import BlackboxFunction
# Create the HyppopyProject class instance
project = HyppopyProject()
project.add_hyperparameter(name="C", domain="uniform", data=[0.0001, 20], dtype="float")
project.add_hyperparameter(name="gamma", domain="uniform", data=[0.0001, 20], dtype="float")
project.add_hyperparameter(name="kernel", domain="categorical", data=["linear", "sigmoid", "poly", "rbf"], dtype="str")
project.add_settings(section="solver", name="max_iterations", value=500)
project.add_settings(section="custom", name="use_solver", value="optunity")
# The BlackboxFunction signature is as follows:
# BlackboxFunction(blackbox_func=None,
# dataloader_func=None,
# preprocess_func=None,
# callback_func=None,
# data=None,
# **kwargs)
#
# - blackbox_func: a function pointer to the users loss function
# - dataloader_func: a function pointer for handling dataloading. The function is called once before
# optimizing. What it returns is passed as first argument to your loss functions
# data argument.
# - preprocess_func: a function pointer for data preprocessing. The function is called once before
# optimizing and gets via kwargs['data'] the raw data object set directly or returned
# from dataloader_func. What this function returns is then what is passed as first
# argument to your loss function.
# - callback_func: a function pointer called after each iteration. The input kwargs is a dictionary
# keeping the parameters used in this iteration, the 'iteration' index, the 'loss'
# and the 'status'. The function in this example is used for realtime printing it's
# input but can also be used for realtime visualization.
# - data: if not done via dataloader_func one can set a raw_data object directly
# - kwargs: dict that whose content is passed to all functions above.
from sklearn.svm import SVC
from sklearn.datasets import load_iris
from sklearn.model_selection import cross_val_score
def my_dataloader_function(**kwargs):
print("Dataloading...")
# kwargs['params'] allows accessing additional parameter passed,
# see below my_preproc_param, my_dataloader_input.
print("my loading argument: {}".format(kwargs['params']['my_dataloader_input']))
iris_data = load_iris()
return [iris_data.data, iris_data.target]
def my_preprocess_function(**kwargs):
print("Preprocessing...")
# kwargs['data'] allows accessing the input data
print("data:", kwargs['data'][0].shape, kwargs['data'][1].shape)
# kwargs['params'] allows accessing additional parameter passed,
# see below my_preproc_param, my_dataloader_input.
print("kwargs['params']['my_preproc_param']={}".format(kwargs['params']['my_preproc_param']), "\n")
# if the preprocessing function returns something,
# the input data will be replaced with the data returned by this function.
x = kwargs['data'][0]
y = kwargs['data'][1]
for i in range(x.shape[0]):
x[i, :] += kwargs['params']['my_preproc_param']
return [x, y]
def my_callback_function(**kwargs):
print("\r{}".format(kwargs), end="")
def my_loss_function(data, params):
clf = SVC(**params)
return -cross_val_score(estimator=clf, X=data[0], y=data[1], cv=3).mean()
# We now create the BlackboxFunction object and pass all function pointers defined above,
# as well as 2 dummy parameter (my_preproc_param, my_dataloader_input) for demonstration purposes.
blackbox = BlackboxFunction(blackbox_func=my_loss_function,
dataloader_func=my_dataloader_function,
preprocess_func=my_preprocess_function,
callback_func=my_callback_function,
my_preproc_param=1,
my_dataloader_input='could/be/a/path')
# Get the solver
solver = SolverPool.get(project=project)
# Give the solver your blackbox
solver.blackbox = blackbox
# Run the solver
solver.run()
# Get your results
df, best = solver.get_results()
print("\n")
print("*"*100)
print("Best Parameter Set:\n{}".format(best))
print("*"*100)
```
#### The Parameter Space Domains
Each hyperparameter needs a range and a domain specifier. The range, specified via 'data', is the left and right bound of an interval (!!!exception is the domain 'categorical', here 'data' is the actual list of data elements!!!) and the domain specifier the way this interval is sampled. Currently supported domains are:
* uniform (samples the interval [a,b] evenly)
* normal (a gaussian sampling of the interval [a,b] such that mu=a+(b-a)/2 and sigma=(b-a)/6)
* loguniform (a logaritmic sampling of the iterval [a,b], such that the exponent e^x is sampled evenly x=[log(a),log(b)])
* categorical (in this case data is not interpreted as interval but as actual list of objects)
One exception is the GridsearchSolver, here we need to specifiy an interval and a number of samples like so: 'data': [a,b,N]. The max_iterations parameter is obsolet in this case because each axis specifies an individual number of samples.
```python
# import the SolverPool class
from hyppopy.solver.GridsearchSolver import GridsearchSolver
# Import the HyppopyProject class
from hyppopy.HyppopyProject import HyppopyProject
# Our function to optimize
def my_loss_func(x, y):
return x**2+y**2
# Creating a HyppopyProject instance
project = HyppopyProject()
project.add_hyperparameter(name="x", domain="uniform", data=[-1.1, 1, 10], dtype="float")
project.add_hyperparameter(name="y", domain="uniform", data=[-1.1, 1, 12], dtype="float")
solver = GridsearchSolver(project=project)
# pass the loss function to the solver
solver.blackbox = my_loss_func
# run the solver
solver.run()
df, best = solver.get_results()
print("\n")
print("*"*100)
print("Best Parameter Set:\n{}".format(best))
print("*"*100)
```
+
+#### Using a Visdom Server to Visualize the Optimization Process
+
+We can simply create a realtime visualization using a visdom server. If installed, start your visdom server via console command:
+```
+>visdom
+```
+
+Go to your browser and open the site: http://localhost:8097
+
+To enable the visualization call the function 'start_viewer' before running the solver:
+
+```python
+#enable visualization
+solver.start_viewer()
+# Run the solver
+solver.run()
+```
+
+You can also change the port and the server name in start_viewer(port=8097, server="http://localhost")
+
diff --git a/examples/solver_comparison.py b/examples/solver_comparison.py
index 9dd2322..66247d1 100644
--- a/examples/solver_comparison.py
+++ b/examples/solver_comparison.py
@@ -1,196 +1,194 @@
# DKFZ
#
#
# Copyright (c) German Cancer Research Center,
# Division of Medical Image Computing.
# All rights reserved.
#
# This software is distributed WITHOUT ANY WARRANTY; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.
#
# See LICENSE
-#
-# Author: Sven Wanner (s.wanner@dkfz.de)
import os
import pickle
import numpy as np
from math import pi
from pprint import pprint
import matplotlib.pyplot as plt
from hyppopy.SolverPool import SolverPool
from hyppopy.HyppopyProject import HyppopyProject
from hyppopy.VirtualFunction import VirtualFunction
from hyppopy.BlackboxFunction import BlackboxFunction
OUTPUTDIR = "C:\\Users\\s635r\\Desktop\\solver_comparison"
SOLVER = ["hyperopt", "optunity", "randomsearch", "optuna"]#, "bayesopt"]
ITERATIONS = [25, 100, 250, 500]
STATREPEATS = 10
VFUNC = "5D3"
OVERWRITE = False
OUTPUTDIR = os.path.join(OUTPUTDIR, VFUNC)
if not os.path.isdir(OUTPUTDIR):
os.makedirs(OUTPUTDIR)
def compute_deviation(solver_name, vfunc_id, iterations, N, fname):
project = HyppopyProject()
project.add_hyperparameter(name="axis_00", domain="uniform", data=[0, 1], dtype="float")
project.add_hyperparameter(name="axis_01", domain="uniform", data=[0, 1], dtype="float")
project.add_hyperparameter(name="axis_02", domain="uniform", data=[0, 1], dtype="float")
project.add_hyperparameter(name="axis_03", domain="uniform", data=[0, 1], dtype="float")
project.add_hyperparameter(name="axis_04", domain="uniform", data=[0, 1], dtype="float")
vfunc = VirtualFunction()
vfunc.load_default(vfunc_id)
minima = vfunc.minima()
def my_loss_function(data, params):
return vfunc(**params)
blackbox = BlackboxFunction(data=[], blackbox_func=my_loss_function)
results = {}
results["gt"] = []
for mini in minima:
results["gt"].append(np.median(mini[0]))
for iter in iterations:
results[iter] = {"minima": {}, "loss": None}
for i in range(vfunc.dims()):
results[iter]["minima"]["axis_0{}".format(i)] = []
project.add_settings(section="solver", name="max_iterations", value=iter)
project.add_settings(section="custom", name="use_solver", value=solver_name)
solver = SolverPool.get(project=project)
solver.blackbox = blackbox
axis_minima = []
best_losses = []
for i in range(vfunc.dims()):
axis_minima.append([])
for n in range(N):
print("\rSolver={} iteration={} round={}".format(solver, iter, n), end="")
solver.run(print_stats=False)
df, best = solver.get_results()
best_row = df['losses'].idxmin()
best_losses.append(df['losses'][best_row])
for i in range(vfunc.dims()):
tmp = df['axis_0{}'.format(i)][best_row]
axis_minima[i].append(tmp)
for i in range(vfunc.dims()):
results[iter]["minima"]["axis_0{}".format(i)] = [np.mean(axis_minima[i]), np.std(axis_minima[i])]
results[iter]["loss"] = [np.mean(best_losses), np.std(best_losses)]
file = open(fname, 'wb')
pickle.dump(results, file)
file.close()
def make_radarplot(results, title, fname=None):
gt = results.pop("gt")
categories = list(results[list(results.keys())[0]]["minima"].keys())
N = len(categories)
angles = [n / float(N) * 2 * pi for n in range(N)]
angles += angles[:1]
ax = plt.subplot(1, 1, 1, polar=True, )
ax.set_theta_offset(pi / 2)
ax.set_theta_direction(-1)
plt.xticks(angles[:-1], categories, color='grey', size=8)
ax.set_rlabel_position(0)
plt.yticks([0.2, 0.4, 0.6, 0.8, 1.0], ["0.2", "0.4", "0.6", "0.8", "1.0"], color="grey", size=7)
plt.ylim(0, 1)
gt += gt[:1]
ax.fill(angles, gt, color=(0.2, 0.8, 0.2), alpha=0.2)
colors = []
cm = plt.get_cmap('Set1')
if len(results) > 2:
indices = list(range(0, len(results) + 1))
indices.pop(2)
else:
indices = list(range(0, len(results)))
for i in range(len(results)):
colors.append(cm(indices[i]))
for iter, data in results.items():
values = []
for i in range(len(categories)):
values.append(data["minima"]["axis_0{}".format(i)][0])
values += values[:1]
color = colors.pop(0)
ax.plot(angles, values, color=color, linewidth=2, linestyle='solid', label="iterations {}".format(iter))
plt.title(title, size=11, color=(0.1, 0.1, 0.1), y=1.1)
plt.legend(bbox_to_anchor=(0.08, 1.12))
if fname is None:
plt.show()
else:
plt.savefig(fname + ".png")
plt.savefig(fname + ".svg")
plt.clf()
def make_deviationerrorplot(fnames):
results = {}
for fname in fnames:
file = open(fname, 'rb')
result = pickle.load(file)
file.close()
results[os.path.basename(fname)] = result
pprint(results)
plt.figure()
for iter in results["hyperopt"].keys():
y = []
if iter == "gt":
x = list(range(len(results["hyperopt"][iter])))
for i in range(len(results["hyperopt"][iter])):
y.append(results["hyperopt"][iter][i])
plt.plot(x, y, "--g", label="groundtruth: {}".format(iter))
continue
x = list(range(len(results["hyperopt"][iter]["minima"])))
for i in range(len(results["hyperopt"][iter]["minima"])):
y.append(results["hyperopt"][iter]["minima"]["axis_0{}".format(i)][0])
plt.plot(x, y, label="iterations: {}".format(iter))
plt.title("")
plt.legend()
plt.show()
##################################################
############### create datasets ##################
fnames = []
for solver_name in SOLVER:
fname = os.path.join(OUTPUTDIR, solver_name)
fnames.append(fname)
if OVERWRITE or not os.path.isfile(fname):
compute_deviation(solver_name, VFUNC, ITERATIONS, N=STATREPEATS, fname=fname)
##################################################
##################################################
##################################################
############## create radarplots #################
for solver_name, fname in zip(SOLVER, fnames):
file = open(fname, 'rb')
results = pickle.load(file)
file.close()
make_radarplot(results, solver_name, fname + "_deviation")
##################################################
##################################################
diff --git a/examples/tutorial_custom_visualization.py b/examples/tutorial_custom_visualization.py
index cf463c1..ab3b706 100644
--- a/examples/tutorial_custom_visualization.py
+++ b/examples/tutorial_custom_visualization.py
@@ -1,107 +1,105 @@
# DKFZ
#
#
# Copyright (c) German Cancer Research Center,
# Division of Medical Image Computing.
# All rights reserved.
#
# This software is distributed WITHOUT ANY WARRANTY; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.
#
# See LICENSE
-#
-# Author: Sven Wanner (s.wanner@dkfz.de)
import matplotlib.pylab as plt
from hyppopy.SolverPool import SolverPool
from hyppopy.HyppopyProject import HyppopyProject
from hyppopy.VirtualFunction import VirtualFunction
from hyppopy.BlackboxFunction import BlackboxFunction
project = HyppopyProject()
project.add_hyperparameter(name="axis_00", domain="uniform", data=[0, 1], dtype="float")
project.add_hyperparameter(name="axis_01", domain="uniform", data=[0, 800], dtype="float")
project.add_hyperparameter(name="axis_02", domain="uniform", data=[0, 5], dtype="float")
project.add_hyperparameter(name="axis_03", domain="uniform", data=[1, 10000], dtype="float")
project.add_hyperparameter(name="axis_04", domain="uniform", data=[0, 10], dtype="float")
project.add_settings(section="solver", name="max_iterations", value=500)
project.add_settings(section="custom", name="use_solver", value="randomsearch")
plt.ion()
fig, axes = plt.subplots(nrows=2, ncols=3, figsize=(12, 8), sharey=True)
plot_data = {"iterations": [],
"loss": [],
"axis_00": [],
"axis_01": [],
"axis_02": [],
"axis_03": [],
"axis_04": []}
def my_visualization_function(**kwargs):
print("\r{}".format(kwargs), end="")
plot_data["iterations"].append(kwargs['iterations'])
plot_data["loss"].append(kwargs['loss'])
plot_data["axis_00"].append(kwargs['axis_00'])
plot_data["axis_01"].append(kwargs['axis_01'])
plot_data["axis_02"].append(kwargs['axis_02'])
plot_data["axis_03"].append(kwargs['axis_03'])
plot_data["axis_04"].append(kwargs['axis_04'])
axes[0, 0].clear()
axes[0, 0].scatter(plot_data["axis_00"], plot_data["loss"], c=plot_data["loss"], cmap="jet", marker='.')
axes[0, 0].set_ylabel("loss")
axes[0, 0].set_xlabel("axis_00")
axes[0, 1].clear()
axes[0, 1].scatter(plot_data["axis_01"], plot_data["loss"], c=plot_data["loss"], cmap="jet", marker='.')
axes[0, 1].set_xlabel("axis_01")
axes[0, 2].clear()
axes[0, 2].scatter(plot_data["axis_02"], plot_data["loss"], c=plot_data["loss"], cmap="jet", marker='.')
axes[0, 2].set_xlabel("axis_02")
axes[1, 0].clear()
axes[1, 0].scatter(plot_data["axis_03"], plot_data["loss"], c=plot_data["loss"], cmap="jet", marker='.')
axes[1, 0].set_ylabel("loss")
axes[1, 0].set_xlabel("axis_03")
axes[1, 1].clear()
axes[1, 1].scatter(plot_data["axis_04"], plot_data["loss"], c=plot_data["loss"], cmap="jet", marker='.')
axes[1, 1].set_xlabel("axis_04")
axes[1, 2].clear()
axes[1, 2].plot(plot_data["iterations"], plot_data["loss"], "--", c=(0.8, 0.8, 0.8, 0.5))
axes[1, 2].scatter(plot_data["iterations"], plot_data["loss"], marker='.', c=(0.2, 0.2, 0.2))
axes[1, 2].set_xlabel("iterations")
plt.draw()
plt.tight_layout()
plt.pause(0.001)
def my_loss_function(data, params):
vfunc = VirtualFunction()
vfunc.load_default(5)
return vfunc(**params)
blackbox = BlackboxFunction(data=[],
blackbox_func=my_loss_function,
callback_func=my_visualization_function)
solver = SolverPool.get(project=project)
solver.blackbox = blackbox
solver.run()
df, best = solver.get_results()
print("\n")
print("*" * 100)
print("Best Parameter Set:\n{}".format(best))
print("*" * 100)
print("")
save_plot = input("Save Plot? [y/n] ")
if save_plot == "y":
plt.savefig('plot_{}.png'.format(project.custom_use_solver))
diff --git a/examples/tutorial_gridsearch.py b/examples/tutorial_gridsearch.py
index a0d872a..cbead25 100644
--- a/examples/tutorial_gridsearch.py
+++ b/examples/tutorial_gridsearch.py
@@ -1,131 +1,129 @@
# DKFZ
#
#
# Copyright (c) German Cancer Research Center,
# Division of Medical Image Computing.
# All rights reserved.
#
# This software is distributed WITHOUT ANY WARRANTY; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.
#
# See LICENSE
-#
-# Author: Sven Wanner (s.wanner@dkfz.de)
# In this tutorial we solve an optimization problem using the GridsearchSolver
# Gridsearch is very inefficient a Randomsearch might most of the time be the
# better choice.
# import the HyppopyProject class keeping track of inputs
from hyppopy.HyppopyProject import HyppopyProject
# import the GridsearchSolver classes
from hyppopy.solver.GridsearchSolver import GridsearchSolver
# import the Blackboxfunction class wrapping your problem for Hyppopy
from hyppopy.BlackboxFunction import BlackboxFunction
# To configure the GridsearchSolver we only need the hyperparameter section. Another
# difference to the other solvers is that we need to define a gridsampling in addition
# to the range: 'data': [0, 1, 100] which means sampling the space from 0 to 1 in 100
# intervals. Gridsearch also supports categorical, uniform, normal and lognormal sampling
config = {
"hyperparameter": {
"C": {
"domain": "uniform",
"data": [0.0001, 20, 20],
"type": "float"
},
"gamma": {
"domain": "uniform",
"data": [0.0001, 20.0, 20],
"type": "float"
},
"kernel": {
"domain": "categorical",
"data": ["linear", "sigmoid", "poly", "rbf"],
"type": "str"
}
},
"settings": {
"solver": {},
"custom": {}
}}
# When creating a HyppopyProject instance we
# pass the config dictionary to the constructor.
project = HyppopyProject(config=config)
# Hyppopy offers a class called BlackboxFunction to wrap your problem for Hyppopy.
# The function signature is as follows:
# BlackboxFunction(blackbox_func=None,
# dataloader_func=None,
# preprocess_func=None,
# callback_func=None,
# data=None,
# **kwargs)
#
# Means we can set a couple of function pointers, a data object and an arbitrary number of custom parameter via kwargs.
#
# - blackbox_func: a function pointer to the actual, user defined, blackbox function that is computing our loss
# - dataloader_func: a function pointer to a function handling the dataloading
# - preprocess_func: a function pointer to a function automatically executed before starting the optimization process
# - callback_func: a function pointer to a function that is called after each iteration with the trail object as input
# - data: setting data can be done via dataloader_func or directly
# - kwargs are passed to all functions above and thus can be used for parameter sharing between the functions
#
# (more details see in the documentation)
#
# Below we demonstrate the usage of all the above by defining a my_dataloader_function which in fact only grabs the
# iris dataset from sklearn and returns it. A my_preprocess_function which also does nothing useful here but
# demonstrating that a custom parameter can be set via kwargs and used in all of our functions when called within
# Hyppopy. The my_callback_function gets as input the dictionary containing the state of the iteration and thus can be
# used to access the current state of each solver iteration. Finally we define the actual loss_function
# my_loss_function, which gets as input a data object and params. Both parameter are fixed, the first is defined by
# the user depending on what is dataloader returns or the data object set in the constructor, the second is a dictionary
# with a sample of your hyperparameter space which content is in the choice of the solver.
from sklearn.svm import SVC
from sklearn.datasets import load_iris
from sklearn.model_selection import cross_val_score
def my_dataloader_function(**kwargs):
print("Dataloading...")
iris_data = load_iris()
return [iris_data.data, iris_data.target]
def my_callback_function(**kwargs):
print("\r{}".format(kwargs), end="")
def my_loss_function(data, params):
clf = SVC(**params)
return -cross_val_score(estimator=clf, X=data[0], y=data[1], cv=3).mean()
# We now create the BlackboxFunction object and pass all function pointers defined above,
# as well as 2 dummy parameter (my_preproc_param, my_dataloader_input) for demonstration purposes.
blackbox = BlackboxFunction(blackbox_func=my_loss_function,
dataloader_func=my_dataloader_function,
callback_func=my_callback_function)
# create a solver instance
solver = GridsearchSolver(project)
# pass the loss function to the solver
solver.blackbox = blackbox
# run the solver
solver.run()
# get the result via get_result() which returns a pandas dataframe
# containing the complete history and a dict best containing the
# best parameter set.
df, best = solver.get_results()
print("\n")
print("*"*100)
print("Best Parameter Set:\n{}".format(best))
print("*"*100)
diff --git a/examples/tutorial_hyppopyprojectclass.py b/examples/tutorial_hyppopyprojectclass.py
index 66d285f..704674f 100644
--- a/examples/tutorial_hyppopyprojectclass.py
+++ b/examples/tutorial_hyppopyprojectclass.py
@@ -1,71 +1,69 @@
# DKFZ
#
#
# Copyright (c) German Cancer Research Center,
# Division of Medical Image Computing.
# All rights reserved.
#
# This software is distributed WITHOUT ANY WARRANTY; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.
#
# See LICENSE
-#
-# Author: Sven Wanner (s.wanner@dkfz.de)
# In this tutorial we demonstrate the HyppopyProject class usage
# import the HyppopyProject class
from hyppopy.HyppopyProject import HyppopyProject
# To configure a solver we need to instanciate a HyppopyProject class.
# This class can be configured using a nested dict. This dict has two
# obligatory sections, hyperparameter and settings. A hyperparameter
# is described using a dict containing a section, data and type field
# and thus the hyperparameter section is a collection of hyperparameter
# dicts. The settings section keeps solver settings. These might depend
# on the solver used and need to be checked for each. E.g. Randomsearch,
# Hyperopt and Optunity need a solver setting max_iterations, the Grid-
# searchSolver don't.
config = {
"hyperparameter": {
"C": {
"domain": "uniform",
"data": [0.0001, 20],
"type": "float"
},
"gamma": {
"domain": "uniform",
"data": [0.0001, 20.0],
"type": "float"
},
"kernel": {
"domain": "categorical",
"data": ["linear", "sigmoid", "poly", "rbf"],
"type": "str"
}
},
"settings": {
"solver": {
"max_iterations": 500
},
"custom": {}
}}
# When creating a HyppopyProject instance we
# pass the config dictionary to the constructor.
project = HyppopyProject(config=config)
# When building the project programmatically we can also use the methods
# add_hyperparameter and add_settings
project.clear()
project.add_hyperparameter(name="C", domain="uniform", data=[0.0001, 20], dtype="float")
project.add_hyperparameter(name="kernel", domain="categorical", data=["linear", "sigmoid"], dtype="str")
project.add_settings(section="solver", name="max_iterations", value=500)
# The custom section can be used freely
project.add_settings(section="custom", name="my_var", value=10)
# Settings are automatically transformed to member variables of the project class with the section as prefix
if project.solver_max_iterations < 1000 and project.custom_my_var == 10:
print("Project configured!")
diff --git a/examples/tutorial_multisolver.py b/examples/tutorial_multisolver.py
index 1dd4637..3d049f1 100644
--- a/examples/tutorial_multisolver.py
+++ b/examples/tutorial_multisolver.py
@@ -1,189 +1,188 @@
# DKFZ
#
#
# Copyright (c) German Cancer Research Center,
# Division of Medical Image Computing.
# All rights reserved.
#
# This software is distributed WITHOUT ANY WARRANTY; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.
#
# See LICENSE
-#
-# Author: Sven Wanner (s.wanner@dkfz.de)
# In this tutorial we solve an optimization problem using the Hyperopt Solver (http://hyperopt.github.io/hyperopt/).
# Hyperopt uses a Baysian - Tree Parzen Estimator - Optimization approach, which means that each iteration computes a
# new function value of the blackbox, interpolates a guess for the whole energy function and predicts a point to
# compute the next function value at. This next point is not necessarily a "better" value, it's only the value with
# the highest uncertainty for the function interpolation.
#
# See a visual explanation e.g. here (http://philipperemy.github.io/visualization/)
# import the HyppopyProject class keeping track of inputs
from hyppopy.HyppopyProject import HyppopyProject
# import the SolverPool singleton class
from hyppopy.SolverPool import SolverPool
# import the Blackboxfunction class wrapping your problem for Hyppopy
from hyppopy.BlackboxFunction import BlackboxFunction
# Next step is defining the problem space and all settings Hyppopy needs to optimize your problem.
# The config is a simple nested dictionary with two obligatory main sections, hyperparameter and settings.
# The hyperparameter section defines your searchspace. Each hyperparameter is again a dictionary with:
#
# - a domain ['categorical', 'uniform', 'normal', 'loguniform']
# - the domain data [left bound, right bound] and
# - a type of your domain ['str', 'int', 'float']
#
# The settings section has two subcategories, solver and custom. The first contains settings for the solver,
# here 'max_iterations' - is the maximum number of iteration.
#
# The custom section allows defining custom parameter. An entry here is transformed to a member variable of the
# HyppopyProject class. These can be useful when implementing new solver classes or for control your hyppopy script.
# Here we use it as a solver switch to control the usage of our solver via the config. This means with the script
# below your can try out every solver by changing use_solver to 'optunity', 'randomsearch', 'gridsearch',...
# It can be used like so: project.custom_use_plugin (see below) If using the gridsearch solver, max_iterations is
# ignored, instead each hyperparameter must specifiy a number of samples additionally to the range like so:
# 'data': [0, 1, 100] which means sampling the space from 0 to 1 in 100 intervals.
config = {
"hyperparameter": {
"C": {
"domain": "uniform",
"data": [0.0001, 20],
"type": "float"
},
"gamma": {
"domain": "uniform",
"data": [0.0001, 20.0],
"type": "float"
},
"kernel": {
"domain": "categorical",
"data": ["linear", "sigmoid", "poly", "rbf"],
"type": "str"
},
"decision_function_shape": {
"domain": "categorical",
"data": ["ovo", "ovr"],
"type": "str"
}
},
"settings": {
"solver": {
- "max_iterations": 200
+ "max_iterations": 500
},
"custom": {
"use_solver": "hyperopt"
}
}}
# When creating a HyppopyProject instance we
# pass the config dictionary to the constructor.
project = HyppopyProject(config=config)
# demonstration of the custom parameter access
print("-"*30)
print("max_iterations:\t{}".format(project.solver_max_iterations))
print("solver chosen -> {}".format(project.custom_use_solver))
print("-"*30)
# The BlackboxFunction signature is as follows:
# BlackboxFunction(blackbox_func=None,
# dataloader_func=None,
# preprocess_func=None,
# callback_func=None,
# data=None,
# **kwargs)
#
# - blackbox_func: a function pointer to the users loss function
# - dataloader_func: a function pointer for handling dataloading. The function is called once before
# optimizing. What it returns is passed as first argument to your loss functions
# data argument.
# - preprocess_func: a function pointer for data preprocessing. The function is called once before
# optimizing and gets via kwargs['data'] the raw data object set directly or returned
# from dataloader_func. What this function returns is then what is passed as first
# argument to your loss function.
# - callback_func: a function pointer called after each iteration. The input kwargs is a dictionary
# keeping the parameters used in this iteration, the 'iteration' index, the 'loss'
# and the 'status'. The function in this example is used for realtime printing it's
# input but can also be used for realtime visualization.
# - data: if not done via dataloader_func one can set a raw_data object directly
# - kwargs: dict that whose content is passed to all functions above.
from sklearn.svm import SVC
from sklearn.datasets import load_iris
from sklearn.model_selection import cross_val_score
def my_dataloader_function(**kwargs):
print("Dataloading...")
# kwargs['params'] allows accessing additional parameter passed, see below my_preproc_param, my_dataloader_input.
print("my loading argument: {}".format(kwargs['params']['my_dataloader_input']))
iris_data = load_iris()
return [iris_data.data, iris_data.target]
def my_preprocess_function(**kwargs):
print("Preprocessing...")
# kwargs['data'] allows accessing the input data
print("data:", kwargs['data'][0].shape, kwargs['data'][1].shape)
# kwargs['params'] allows accessing additional parameter passed, see below my_preproc_param, my_dataloader_input.
print("kwargs['params']['my_preproc_param']={}".format(kwargs['params']['my_preproc_param']), "\n")
# if the preprocessing function returns something,
# the input data will be replaced with the data returned by this function.
x = kwargs['data'][0]
y = kwargs['data'][1]
for i in range(x.shape[0]):
x[i, :] += kwargs['params']['my_preproc_param']
return [x, y]
def my_callback_function(**kwargs):
print("\r{}".format(kwargs), end="")
def my_loss_function(data, params):
clf = SVC(**params)
return -cross_val_score(estimator=clf, X=data[0], y=data[1], cv=3).mean()
# We now create the BlackboxFunction object and pass all function pointers defined above,
# as well as 2 dummy parameter (my_preproc_param, my_dataloader_input) for demonstration purposes.
blackbox = BlackboxFunction(blackbox_func=my_loss_function,
dataloader_func=my_dataloader_function,
preprocess_func=my_preprocess_function,
callback_func=my_callback_function,
my_preproc_param=1,
my_dataloader_input='could/be/a/path')
# Last step, is we use our SolverPool which automatically returns the correct solver.
# There are multiple ways to get the desired solver from the solver pool.
# 1. solver = SolverPool.get('hyperopt')
# solver.project = project
# 2. solver = SolverPool.get('hyperopt', project)
# 3. The SolverPool will look for the field 'use_solver' in the project instance, if
# it is present it will be used to specify the solver so that in this case it is enough
# to pass the project instance.
solver = SolverPool.get(project=project)
# Give the solver your blackbox and run it. After execution we can get the result
# via get_result() which returns a pandas dataframe containing the complete history
# The dict best contains the best parameter set.
solver.blackbox = blackbox
+#solver.start_viewer()
solver.run()
df, best = solver.get_results()
print("\n")
print("*"*100)
print("Best Parameter Set:\n{}".format(best))
print("*"*100)
diff --git a/examples/tutorial_simple.py b/examples/tutorial_simple.py
index 6e24fae..43dead1 100644
--- a/examples/tutorial_simple.py
+++ b/examples/tutorial_simple.py
@@ -1,86 +1,84 @@
# DKFZ
#
#
# Copyright (c) German Cancer Research Center,
# Division of Medical Image Computing.
# All rights reserved.
#
# This software is distributed WITHOUT ANY WARRANTY; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.
#
# See LICENSE
-#
-# Author: Sven Wanner (s.wanner@dkfz.de)
# A hyppopy minimal example optimizing a simple demo function f(x,y) = x**2+y**2
# import the HyppopyProject class keeping track of inputs
from hyppopy.HyppopyProject import HyppopyProject
# import the HyperoptSolver class
from hyppopy.solver.HyperoptSolver import HyperoptSolver
# To configure the Hyppopy solver we use a simple nested dictionary with two obligatory main sections,
# hyperparameter and settings. The hyperparameter section defines your searchspace. Each hyperparameter
# is again a dictionary with:
#
# - a domain ['categorical', 'uniform', 'normal', 'loguniform']
# - the domain data [left bound, right bound] and
# - a type of your domain ['str', 'int', 'float']
#
# The settings section has two subcategories, solver and custom. The first contains settings for the solver,
# here 'max_iterations' - is the maximum number of iteration.
#
# The custom section allows defining custom parameter. An entry here is transformed to a member variable of the
# HyppopyProject class. These can be useful when implementing new solver classes or for control your hyppopy script.
# Here we use it as a solver switch to control the usage of our solver via the config. This means with the script
# below your can try out every solver by changing use_solver to 'optunity', 'randomsearch', 'gridsearch',...
# It can be used like so: project.custom_use_plugin (see below) If using the gridsearch solver, max_iterations is
# ignored, instead each hyperparameter must specifiy a number of samples additionally to the range like so:
# 'data': [0, 1, 100] which means sampling the space from 0 to 1 in 100 intervals.
config = {
"hyperparameter": {
"x": {
"domain": "normal",
"data": [-10.0, 10.0],
"type": "float"
},
"y": {
"domain": "uniform",
"data": [-10.0, 10.0],
"type": "float"
}
},
"settings": {
"solver": {
"max_iterations": 500
},
"custom": {}
}}
# When creating a HyppopyProject instance we
# pass the config dictionary to the constructor.
project = HyppopyProject(config=config)
# The user defined loss function
def my_loss_function(x, y):
return x**2+y**2
# create a solver instance
solver = HyperoptSolver(project)
# pass the loss function to the solver
solver.blackbox = my_loss_function
# run the solver
solver.run()
df, best = solver.get_results()
print("\n")
print("*"*100)
print("Best Parameter Set:\n{}".format(best))
print("*"*100)
diff --git a/hyppopy/BlackboxFunction.py b/hyppopy/BlackboxFunction.py
index 8fce552..1348d4f 100644
--- a/hyppopy/BlackboxFunction.py
+++ b/hyppopy/BlackboxFunction.py
@@ -1,98 +1,96 @@
# DKFZ
#
#
# Copyright (c) German Cancer Research Center,
# Division of Medical Image Computing.
# All rights reserved.
#
# This software is distributed WITHOUT ANY WARRANTY; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.
#
# See LICENSE
-#
-# Author: Sven Wanner (s.wanner@dkfz.de)
import os
import logging
import functools
from hyppopy.globals import DEBUGLEVEL
LOG = logging.getLogger(os.path.basename(__file__))
LOG.setLevel(DEBUGLEVEL)
def default_kwargs(**defaultKwargs):
def actual_decorator(fn):
@functools.wraps(fn)
def g(*args, **kwargs):
defaultKwargs.update(kwargs)
return fn(*args, **defaultKwargs)
return g
return actual_decorator
class BlackboxFunction(object):
@default_kwargs(blackbox_func=None, dataloader_func=None, preprocess_func=None, callback_func=None, data=None)
def __init__(self, **kwargs):
self._blackbox_func = None
self._preprocess_func = None
self._dataloader_func = None
self._callback_func = None
self._raw_data = None
self._data = None
self.setup(kwargs)
def __call__(self, **kwargs):
return self.blackbox_func(self.data, kwargs)
def setup(self, kwargs):
self._blackbox_func = kwargs['blackbox_func']
self._preprocess_func = kwargs['preprocess_func']
self._dataloader_func = kwargs['dataloader_func']
self._callback_func = kwargs['callback_func']
self._raw_data = kwargs['data']
self._data = self._raw_data
del kwargs['blackbox_func']
del kwargs['preprocess_func']
del kwargs['dataloader_func']
del kwargs['data']
params = kwargs
if self.dataloader_func is not None:
self._raw_data = self.dataloader_func(params=params)
assert self._raw_data is not None, "Missing data exception!"
assert self.blackbox_func is not None, "Missing blackbox fucntion exception!"
if self.preprocess_func is not None:
result = self.preprocess_func(data=self._raw_data, params=params)
if result is not None:
self._data = result
else:
self._data = self._raw_data
else:
self._data = self._raw_data
@property
def blackbox_func(self):
return self._blackbox_func
@property
def preprocess_func(self):
return self._preprocess_func
@property
def dataloader_func(self):
return self._dataloader_func
@property
def callback_func(self):
return self._callback_func
@property
def raw_data(self):
return self._raw_data
@property
def data(self):
return self._data
diff --git a/hyppopy/HyppopyProject.py b/hyppopy/HyppopyProject.py
index 19409fa..ef5f22b 100644
--- a/hyppopy/HyppopyProject.py
+++ b/hyppopy/HyppopyProject.py
@@ -1,103 +1,101 @@
# DKFZ
#
#
# Copyright (c) German Cancer Research Center,
# Division of Medical Image Computing.
# All rights reserved.
#
# This software is distributed WITHOUT ANY WARRANTY; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.
#
# See LICENSE
-#
-# Author: Sven Wanner (s.wanner@dkfz.de)
import warnings
from hyppopy.globals import *
LOG = logging.getLogger(os.path.basename(__file__))
LOG.setLevel(DEBUGLEVEL)
class HyppopyProject(object):
def __init__(self, config=None):
self._hyperparameter = {}
self._settings = {}
self._extmembers = []
if config is not None:
self.set_config(config)
def clear(self):
self._hyperparameter = {}
self._settings = {}
for added in self._extmembers:
if added in self.__dict__.keys():
del self.__dict__[added]
self._extmembers = []
def set_config(self, config):
self.clear()
assert isinstance(config, dict), "Input Error, config of type {} not supported!".format(type(config))
assert HYPERPARAMETERPATH in config.keys(), "Missing hyperparameter section in config dict"
#assert SETTINGSPATH in config.keys(), "Missing settings section in config dict"
if not SETTINGSPATH in config.keys():
config[SETTINGSPATH] = {"solver": {"max_iterations": DEFAULTITERATIONS}}
msg = "config dict had no section {0}/solver/max_iterations, set default value: {1}".format(SETTINGSPATH, DEFAULTITERATIONS)
warnings.warn(msg)
LOG.warning(msg)
self._hyperparameter = config[HYPERPARAMETERPATH]
self._settings = config[SETTINGSPATH]
self.parse_members()
def add_hyperparameter(self, name, domain, data, dtype):
assert isinstance(name, str), "precondition violation, name of type {} not allowed, expect str!".format(type(name))
assert isinstance(domain, str), "precondition violation, domain of type {} not allowed, expect str!".format(type(domain))
assert domain in SUPPORTED_DOMAINS, "domain {} not supported, expect {}!".format(domain, SUPPORTED_DOMAINS)
assert isinstance(data, list) or isinstance(data, tuple), "precondition violation, data of type {} not allowed, expect list or tuple!".format(type(data))
if domain != "categorical":
assert len(data) == 3 or len(data) == 2, "precondition violation, data must be a list of len 2 or 3"
assert isinstance(dtype, str), "precondition violation, dtype of type {} not allowed, expect str!".format(type(dtype))
assert dtype in SUPPORTED_DTYPES, "precondition violation, dtype {} not supported, expect {}!".format(dtype, SUPPORTED_DTYPES)
self._hyperparameter[name] = {"domain": domain, "data": data, "type": dtype}
def add_settings(self, section, name, value):
assert isinstance(section, str), "precondition violation, section of type {} not allowed, expect str!".format(type(section))
assert isinstance(name, str), "precondition violation, name of type {} not allowed, expect str!".format(type(name))
if section not in self._settings.keys():
self._settings[section] = {}
self._settings[section][name] = value
self.parse_members()
def parse_members(self):
for section_name, content in self.settings.items():
for name, value in content.items():
member_name = section_name + "_" + name
if member_name not in self._extmembers:
setattr(self, member_name, value)
self._extmembers.append(member_name)
else:
self.__dict__[member_name] = value
def get_typeof(self, hyperparametername):
if not hyperparametername in self.hyperparameter.keys():
return None
dtype = self.hyperparameter[hyperparametername]["type"]
if dtype == 'str':
return str
if dtype == 'int':
return int
if dtype == 'float':
return float
@property
def hyperparameter(self):
return self._hyperparameter
@property
def settings(self):
return self._settings
diff --git a/hyppopy/ProjectManager.py b/hyppopy/ProjectManager.py
index 4257bf8..0d072dd 100644
--- a/hyppopy/ProjectManager.py
+++ b/hyppopy/ProjectManager.py
@@ -1,69 +1,67 @@
# DKFZ
#
#
# Copyright (c) German Cancer Research Center,
# Division of Medical Image Computing.
# All rights reserved.
#
# This software is distributed WITHOUT ANY WARRANTY; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.
#
# See LICENSE
-#
-# Author: Sven Wanner (s.wanner@dkfz.de)
from .Singleton import *
import os
import logging
from hyppopy.HyppopyProject import HyppopyProject
from hyppopy.globals import DEBUGLEVEL
LOG = logging.getLogger(os.path.basename(__file__))
LOG.setLevel(DEBUGLEVEL)
@singleton_object
class ProjectManager(metaclass=Singleton):
def __init__(self):
self._current_project = None
self._projects = {}
def clear_all(self):
pass
def new_project(self, name="HyppopyProject", config=None):
if name in self._projects.keys():
name = self.check_projectname(name)
self._projects[name] = HyppopyProject(config)
self._current_project = self._projects[name]
return self._current_project
def check_projectname(self, name):
split = name.split(".")
if len(split) == 0:
return split[0] + "." + str(0).zfill(3)
else:
try:
number = int(split[-1])
del split[-1]
except:
number = 0
return '.'.join(split) + "." + str(number).zfill(3)
def get_current(self):
if self._current_project is None:
self.new_project()
return self._current_project
def get_project(self, name):
if name in self._projects.keys():
self._current_project = self._projects[name]
return self.get_current()
return self.new_project(name)
def get_projectnames(self):
return self._projects.keys()
diff --git a/hyppopy/Singleton.py b/hyppopy/Singleton.py
index 90575de..fac0bc8 100644
--- a/hyppopy/Singleton.py
+++ b/hyppopy/Singleton.py
@@ -1,52 +1,50 @@
# DKFZ
#
#
# Copyright (c) German Cancer Research Center,
# Division of Medical Image Computing.
# All rights reserved.
#
# This software is distributed WITHOUT ANY WARRANTY; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.
#
# See LICENSE
-#
-# Author: Sven Wanner (s.wanner@dkfz.de)
class Singleton(type):
_instances = {}
def __call__(cls, *args, **kwargs):
if cls not in cls._instances:
cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
return cls._instances[cls]
@classmethod
def __instancecheck__(mcs, instance):
if instance.__class__ is mcs:
return True
else:
return isinstance(instance.__class__, mcs)
def singleton_object(cls):
"""Class decorator that transforms (and replaces) a class definition (which
must have a Singleton metaclass) with the actual singleton object. Ensures
that the resulting object can still be "instantiated" (i.e., called),
returning the same object. Also ensures the object can be pickled, is
hashable, and has the correct string representation (the name of the
singleton)
"""
assert isinstance(cls, Singleton), cls.__name__ + " must use Singleton metaclass"
def self_instantiate(self):
return self
cls.__call__ = self_instantiate
cls.__hash__ = lambda self: hash(cls)
cls.__repr__ = lambda self: cls.__name__
cls.__reduce__ = lambda self: cls.__name__
obj = cls()
obj.__name__ = cls.__name__
return obj
diff --git a/hyppopy/Solver/BayesOptSolver.py b/hyppopy/Solver/BayesOptSolver.py
index 338cb8b..5a58bdf 100644
--- a/hyppopy/Solver/BayesOptSolver.py
+++ b/hyppopy/Solver/BayesOptSolver.py
@@ -1,84 +1,82 @@
# DKFZ
#
#
# Copyright (c) German Cancer Research Center,
# Division of Medical Image Computing.
# All rights reserved.
#
# This software is distributed WITHOUT ANY WARRANTY; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.
#
# See LICENSE
-#
-# Author: Sven Wanner (s.wanner@dkfz.de)
import os
import logging
import warnings
import numpy as np
from pprint import pformat
from hyperopt import Trials
from bayes_opt import BayesianOptimization
from hyppopy.globals import DEBUGLEVEL
from hyppopy.solver.HyppopySolver import HyppopySolver
LOG = logging.getLogger(os.path.basename(__file__))
LOG.setLevel(DEBUGLEVEL)
class BayesOptSolver(HyppopySolver):
def __init__(self, project=None):
HyppopySolver.__init__(self, project)
self._searchspace = None
self._idx = None
def reformat_parameter(self, params):
out_params = {}
for name, value in params.items():
if self._searchspace[name]["domain"] == "categorical":
out_params[name] = self._searchspace[name]["data"][int(np.round(value))]
else:
if self._searchspace[name]["type"] == "int":
out_params[name] = int(np.round(value))
else:
out_params[name] = value
return out_params
def loss_function_call(self, params):
params = self.reformat_parameter(params)
for key in params.keys():
if self.project.get_typeof(key) is int:
params[key] = int(round(params[key]))
return self.blackbox(**params)
def execute_solver(self, searchspace):
LOG.debug("execute_solver using solution space:\n\n\t{}\n".format(pformat(searchspace)))
self.trials = Trials()
self._idx = 0
try:
optimizer = BayesianOptimization(f=self.loss_function, pbounds=searchspace, verbose=0)
optimizer.maximize(init_points=2, n_iter=self.max_iterations)
self.best = self.reformat_parameter(optimizer.max["params"])
except Exception as e:
LOG.error("internal error in bayes_opt maximize occured. {}".format(e))
raise BrokenPipeError("internal error in bayes_opt maximize occured. {}".format(e))
def convert_searchspace(self, hyperparameter):
LOG.debug("convert input parameter\n\n\t{}\n".format(pformat(hyperparameter)))
self._searchspace = hyperparameter
pbounds = {}
for name, param in hyperparameter.items():
if param["domain"] != "categorical":
if param["domain"] != "uniform":
msg = "Warning: BayesOpt cannot handle {} domain. Only uniform and categorical domains are supported!".format(
param["domain"])
warnings.warn(msg)
LOG.warning(msg)
pbounds[name] = (param["data"][0], param["data"][1])
else:
pbounds[name] = (0, len(param["data"])-1)
return pbounds
diff --git a/hyppopy/Solver/GridsearchSolver.py b/hyppopy/Solver/GridsearchSolver.py
index d0a6604..272482a 100644
--- a/hyppopy/Solver/GridsearchSolver.py
+++ b/hyppopy/Solver/GridsearchSolver.py
@@ -1,182 +1,180 @@
# DKFZ
#
#
# Copyright (c) German Cancer Research Center,
# Division of Medical Image Computing.
# All rights reserved.
#
# This software is distributed WITHOUT ANY WARRANTY; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.
#
# See LICENSE
-#
-# Author: Sven Wanner (s.wanner@dkfz.de)
import os
import logging
import numpy as np
from pprint import pformat
from scipy.stats import norm
from itertools import product
from hyppopy.globals import DEBUGLEVEL
from hyppopy.solver.HyppopySolver import HyppopySolver
LOG = logging.getLogger(os.path.basename(__file__))
LOG.setLevel(DEBUGLEVEL)
def get_uniform_axis_sample(a, b, N, dtype):
"""
returns a uniform sample x(n) in the range [a,b] sampled at N pojnts
:param a: left value range bound
:param b: right value range bound
:param N: discretization of intervall [a,b]
:param dtype: data type
:return: [list] axis range
"""
assert a < b, "condition a < b violated!"
assert isinstance(N, int), "condition N of type int violated!"
assert isinstance(dtype, str), "condition type of type str violated!"
if dtype == "int":
return list(np.linspace(a, b, N).astype(int))
elif dtype == "float" or dtype == "double":
return list(np.linspace(a, b, N))
else:
raise AssertionError("dtype {} not supported for uniform sampling!".format(dtype))
def get_norm_cdf(N):
"""
returns a normed gaussian cdf (range [0,1]) with N sampling points
:param N: sampling points
:return: [ndarray] gaussian cdf function values
"""
assert isinstance(N, int), "condition N of type int violated!"
even = True
if N % 2 != 0:
N -= 1
even = False
N = int(N/2)
sigma = 1/3
x = np.linspace(0, 1, N)
y1 = norm.cdf(x, loc=0, scale=sigma)-0.5
if not even:
y1 = np.append(y1, [0.5])
y2 = 1-(norm.cdf(x, loc=0, scale=sigma)-0.5)
y2 = np.flip(y2, axis=0)
y = np.concatenate((y1, y2), axis=0)
return y
def get_gaussian_axis_sample(a, b, N, dtype):
"""
returns a function value f(n) where f is a gaussian cdf in range [a, b] and N sampling points
:param a: left value range bound
:param b: right value range bound
:param N: discretization of intervall [a,b]
:param dtype: data type
:return: [list] axis range
"""
assert a < b, "condition a < b violated!"
assert isinstance(N, int), "condition N of type int violated!"
assert isinstance(dtype, str), "condition type of type str violated!"
data = []
for n in range(N):
x = a + get_norm_cdf(N)[n]*(b-a)
if dtype == "int":
data.append(int(x))
elif dtype == "float" or dtype == "double":
data.append(x)
else:
raise AssertionError("dtype {} not supported for uniform sampling!".format(dtype))
return data
def get_logarithmic_axis_sample(a, b, N, dtype):
"""
returns a function value f(n) where f is logarithmic function e^x sampling
the exponent range [log(a), log(b)] linear at N sampling points.
The function values returned are in the range [a, b].
:param a: left value range bound
:param b: right value range bound
:param N: discretization of intervall [a,b]
:param dtype: data type
:return: [list] axis range
"""
assert a < b, "condition a < b violated!"
assert a > 0, "condition a > 0 violated!"
assert isinstance(N, int), "condition N of type int violated!"
assert isinstance(dtype, str), "condition type of type str violated!"
# convert input range into exponent range
lexp = np.log(a)
rexp = np.log(b)
exp_range = np.linspace(lexp, rexp, N)
data = []
for n in range(exp_range.shape[0]):
x = np.exp(exp_range[n])
if dtype == "int":
data.append(int(x))
elif dtype == "float" or dtype == "double":
data.append(x)
else:
raise AssertionError("dtype {} not supported for uniform sampling!".format(dtype))
return data
class GridsearchSolver(HyppopySolver):
"""
The GridsearchSolver class implements a gridsearch optimization. The gridsearch supports
categorical, uniform, normal and loguniform sampling. To use the GridsearchSolver, besides
a range, one must specifiy the number of samples in the domain, e.g. 'data': [0, 1, 100]
"""
def __init__(self, project=None):
HyppopySolver.__init__(self, project)
self._has_maxiteration_field = False
def loss_function_call(self, params):
loss = self.blackbox(**params)
if loss is None:
return np.nan
return loss
def execute_solver(self, searchspace):
for x in product(*searchspace[1]):
params = {}
for name, value in zip(searchspace[0], x):
params[name] = value
try:
self.loss_function(**params)
except Exception as e:
msg = "internal error in randomsearch execute_solver occured. {}".format(e)
LOG.error(msg)
raise BrokenPipeError(msg)
self.best = self._trials.argmin
def convert_searchspace(self, hyperparameter):
"""
the function converts the standard parameter input into a range list depending
on the domain. These rangelists are later used with itertools product to create
a paramater space sample of each combination.
:param hyperparameter: [dict] hyperparameter space
:return: [list] name and range for each parameter space axis
"""
LOG.debug("convert input parameter\n\n\t{}\n".format(pformat(hyperparameter)))
searchspace = [[], []]
for name, param in hyperparameter.items():
if param["domain"] == "categorical":
searchspace[0].append(name)
searchspace[1].append(param["data"])
elif param["domain"] == "uniform":
searchspace[0].append(name)
searchspace[1].append(get_uniform_axis_sample(param["data"][0], param["data"][1], param["data"][2], param["type"]))
elif param["domain"] == "normal":
searchspace[0].append(name)
searchspace[1].append(get_gaussian_axis_sample(param["data"][0], param["data"][1], param["data"][2], param["type"]))
elif param["domain"] == "loguniform":
searchspace[0].append(name)
searchspace[1].append(get_logarithmic_axis_sample(param["data"][0], param["data"][1], param["data"][2], param["type"]))
return searchspace
diff --git a/hyppopy/Solver/HyperoptSolver.py b/hyppopy/Solver/HyperoptSolver.py
index b9b91ff..19ec8ee 100644
--- a/hyppopy/Solver/HyperoptSolver.py
+++ b/hyppopy/Solver/HyperoptSolver.py
@@ -1,158 +1,164 @@
# DKFZ
#
#
# Copyright (c) German Cancer Research Center,
# Division of Medical Image Computing.
# All rights reserved.
#
# This software is distributed WITHOUT ANY WARRANTY; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.
#
# See LICENSE
-#
-# Author: Sven Wanner (s.wanner@dkfz.de)
import os
import copy
import logging
import numpy as np
from pprint import pformat
from hyperopt import fmin, tpe, hp, STATUS_OK, STATUS_FAIL, Trials
from hyppopy.globals import DEBUGLEVEL
from hyppopy.solver.HyppopySolver import HyppopySolver
from hyppopy.BlackboxFunction import BlackboxFunction
LOG = logging.getLogger(os.path.basename(__file__))
LOG.setLevel(DEBUGLEVEL)
class HyperoptSolver(HyppopySolver):
def __init__(self, project=None):
HyppopySolver.__init__(self, project)
self._searchspace = None
def loss_function(self, params):
for name, p in self._searchspace.items():
if p["domain"] != "categorical":
if params[name] < p["data"][0]:
params[name] = p["data"][0]
if params[name] > p["data"][1]:
params[name] = p["data"][1]
status = STATUS_FAIL
try:
loss = self.blackbox(**params)
if loss is not None:
status = STATUS_OK
else:
loss = 1e9
except Exception as e:
LOG.error("execution of self.blackbox(**params) failed due to:\n {}".format(e))
status = STATUS_FAIL
loss = 1e9
+ cbd = copy.deepcopy(params)
+ cbd['iterations'] = self._trials.trials[-1]['tid'] + 1
+ cbd['loss'] = loss
+ cbd['status'] = status
+ cbd['book_time'] = self._trials.trials[-1]['book_time']
+ cbd['refresh_time'] = self._trials.trials[-1]['refresh_time']
if isinstance(self.blackbox, BlackboxFunction) and self.blackbox.callback_func is not None:
- cbd = copy.deepcopy(params)
- cbd['iterations'] = self._trials.trials[-1]['tid'] + 1
- cbd['loss'] = loss
- cbd['status'] = status
+ # cbd = copy.deepcopy(params)
+ # cbd['iterations'] = self._trials.trials[-1]['tid'] + 1
+ # cbd['loss'] = loss
+ # cbd['status'] = status
self.blackbox.callback_func(**cbd)
+ if self._visdom_viewer is not None:
+ self._visdom_viewer.update(cbd)
return {'loss': loss, 'status': status}
def execute_solver(self, searchspace):
LOG.debug("execute_solver using solution space:\n\n\t{}\n".format(pformat(searchspace)))
self.trials = Trials()
try:
self.best = fmin(fn=self.loss_function,
space=searchspace,
algo=tpe.suggest,
max_evals=self.max_iterations,
trials=self.trials)
except Exception as e:
msg = "internal error in hyperopt.fmin occured. {}".format(e)
LOG.error(msg)
raise BrokenPipeError(msg)
def convert_searchspace(self, hyperparameter):
self._searchspace = hyperparameter
solution_space = {}
for name, content in hyperparameter.items():
param_settings = {'name': name}
for key, value in content.items():
if key == 'domain':
param_settings['domain'] = value
elif key == 'data':
param_settings['data'] = value
elif key == 'type':
param_settings['dtype'] = value
solution_space[name] = self.convert(param_settings)
return solution_space
def convert(self, param_settings):
name = param_settings["name"]
domain = param_settings["domain"]
dtype = param_settings["dtype"]
data = param_settings["data"]
assert isinstance(data, list), "precondition violation. data of type {} not allowed!".format(type(data))
assert len(data) >= 2, "precondition violation, data must be of length 2, [left_bound, right_bound]"
assert isinstance(domain, str), "precondition violation. domain of type {} not allowed!".format(type(domain))
assert isinstance(dtype, str), "precondition violation. dtype of type {} not allowed!".format(type(dtype))
if domain == "uniform":
if dtype == "float" or dtype == "double":
return hp.uniform(name, data[0], data[1])
elif dtype == "int":
data = list(np.arange(int(data[0]), int(data[1] + 1)))
return hp.choice(name, data)
else:
msg = "cannot convert the type {} in domain {}".format(dtype, domain)
LOG.error(msg)
raise LookupError(msg)
elif domain == "loguniform":
if dtype == "float" or dtype == "double":
if data[0] == 0:
data[0] += 1e-23
assert data[0] > 0, "precondition Violation, a < 0!"
assert data[0] < data[1], "precondition Violation, a > b!"
assert data[1] > 0, "precondition Violation, b < 0!"
lexp = np.log(data[0])
rexp = np.log(data[1])
assert lexp is not np.nan, "precondition violation, left bound input error, results in nan!"
assert rexp is not np.nan, "precondition violation, right bound input error, results in nan!"
return hp.loguniform(name, lexp, rexp)
else:
msg = "cannot convert the type {} in domain {}".format(dtype, domain)
LOG.error(msg)
raise LookupError(msg)
elif domain == "normal":
if dtype == "float" or dtype == "double":
mu = (data[1] - data[0]) / 2.0
sigma = mu / 3
return hp.normal(name, data[0] + mu, sigma)
else:
msg = "cannot convert the type {} in domain {}".format(dtype, domain)
LOG.error(msg)
raise LookupError(msg)
elif domain == "categorical":
if dtype == 'str':
return hp.choice(name, data)
elif dtype == 'bool':
data = []
for elem in data:
if elem == "true" or elem == "True" or elem == 1 or elem == "1":
data.append(True)
elif elem == "false" or elem == "False" or elem == 0 or elem == "0":
data.append(False)
else:
msg = "cannot convert the type {} in domain {}, unknown bool type value".format(dtype, domain)
LOG.error(msg)
raise LookupError(msg)
return hp.choice(name, data)
else:
msg = "Precondition violation, domain named {} not available!".format(domain)
LOG.error(msg)
raise IOError(msg)
diff --git a/hyppopy/Solver/HyppopySolver.py b/hyppopy/Solver/HyppopySolver.py
index 7d59869..864985d 100644
--- a/hyppopy/Solver/HyppopySolver.py
+++ b/hyppopy/Solver/HyppopySolver.py
@@ -1,329 +1,342 @@
# DKFZ
#
#
# Copyright (c) German Cancer Research Center,
# Division of Medical Image Computing.
# All rights reserved.
#
# This software is distributed WITHOUT ANY WARRANTY; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.
#
# See LICENSE
-#
-# Author: Sven Wanner (s.wanner@dkfz.de)
import abc
import os
import copy
import types
import logging
import datetime
import numpy as np
import pandas as pd
from hyperopt import Trials
from hyppopy.globals import DEBUGLEVEL
+from hyppopy.VisdomViewer import VisdomViewer
from hyppopy.HyppopyProject import HyppopyProject
from hyppopy.BlackboxFunction import BlackboxFunction
from hyppopy.VirtualFunction import VirtualFunction
from hyppopy.globals import DEBUGLEVEL, DEFAULTITERATIONS
LOG = logging.getLogger(os.path.basename(__file__))
LOG.setLevel(DEBUGLEVEL)
class HyppopySolver(object):
"""
The HyppopySolver class is the base class for all solver addons. It defines virtual functions a child class has
to implement to deal with the front-end communication, orchestrating the optimization process and ensuring a proper
process information storing.
The key idea is that the HyppopySolver class defines an interface to configure and run an object instance of itself
independently from the concrete solver lib used to optimize in the background. To achieve this goal an addon
developer needs to implement the abstract methods 'convert_searchspace', 'execute_solver' and 'loss_function_call'.
These methods abstract the peculiarities of the solver libs to offer, on the user side, a simple and consistent
parameter space configuration and optimization procedure. The method 'convert_searchspace' transforms the hyppopy
parameter space description into the solver lib specific description. The method loss_function_call is used to
handle solver lib specifics of calling the actual blackbox function and execute_solver is executed when the run
method is invoked und takes care of calling the solver lib solving routine.
"""
def __init__(self, project=None):
self._idx = None
self._best = None
self._trials = None
self._blackbox = None
self._max_iterations = None
self._project = project
self._total_duration = None
self._solver_overhead = None
self._time_per_iteration = None
self._accumulated_blackbox_time = None
self._has_maxiteration_field = True
+ self._visdom_viewer = None
@abc.abstractmethod
def convert_searchspace(self, hyperparameter):
"""
This function gets the unified hyppopy-like parameterspace description as input and, if necessary, should
convert it into a solver lib specific format. The function is invoked when run is called and what it returns
is passed as searchspace argument to the function execute_solver.
:param hyperparameter: [dict] nested parameter description dict e.g. {'name': {'domain':'uniform', 'data':[0,1], 'type':'float'}, ...}
:return: [object] converted hyperparameter space
"""
raise NotImplementedError('users must define convert_searchspace to use this class')
@abc.abstractmethod
def execute_solver(self, searchspace):
"""
This function is called immediatly after convert_searchspace and get the output of the latter as input. It's
purpose is to call the solver libs main optimization function.
:param searchspace: converted hyperparameter space
"""
raise NotImplementedError('users must define execute_solver to use this class')
@abc.abstractmethod
def loss_function_call(self, params):
"""
This function is called within the function loss_function and encapsulates the actual blackbox function call
in each iteration. The function loss_function takes care of the iteration driving and reporting, but each solver
lib might need some special treatment between the parameter set selection and the calling of the actual blackbox
function, e.g. parameter converting.
:param params: [dict] hyperparameter space sample e.g. {'p1': 0.123, 'p2': 3.87, ...}
:return: [float] loss
"""
raise NotImplementedError('users must define convert_searchspace to use this class')
def loss_function(self, **params):
"""
This function is called each iteration with a selected parameter set. The parameter set selection is driven by
the solver lib itself. The purpose of this function is to take care of the iteration reporting and the calling
of the callback_func is available. As a developer you might want to overwrite this function completely (e.g.
HyperoptSolver) but then you need to take care for iteration reporting for yourself. The alternative is to only
implement loss_function_call (e.g. OptunitySolver).
:param params: [dict] hyperparameter space sample e.g. {'p1': 0.123, 'p2': 3.87, ...}
:return: [float] loss
"""
self._idx += 1
vals = {}
idx = {}
for key, value in params.items():
vals[key] = [value]
idx[key] = [self._idx]
trial = {'tid': self._idx,
'result': {'loss': None, 'status': 'ok'},
'misc': {
'tid': self._idx,
'idxs': idx,
'vals': vals
},
'book_time': datetime.datetime.now(),
'refresh_time': None
}
try:
loss = self.loss_function_call(params)
trial['result']['loss'] = loss
trial['result']['status'] = 'ok'
if loss == np.nan:
trial['result']['status'] = 'failed'
except Exception as e:
LOG.error("computing loss failed due to:\n {}".format(e))
loss = np.nan
trial['result']['loss'] = np.nan
trial['result']['status'] = 'failed'
trial['refresh_time'] = datetime.datetime.now()
self._trials.trials.append(trial)
+ cbd = copy.deepcopy(params)
+ cbd['iterations'] = self._idx
+ cbd['loss'] = loss
+ cbd['status'] = trial['result']['status']
+ cbd['book_time'] = trial['book_time']
+ cbd['refresh_time'] = trial['refresh_time']
if isinstance(self.blackbox, BlackboxFunction) and self.blackbox.callback_func is not None:
- cbd = copy.deepcopy(params)
- cbd['iterations'] = self._idx
- cbd['loss'] = loss
- cbd['status'] = trial['result']['status']
self.blackbox.callback_func(**cbd)
+ if self._visdom_viewer is not None:
+ self._visdom_viewer.update(cbd)
return loss
def run(self, print_stats=True):
"""
This function starts the optimization process.
:param print_stats: [bool] en- or disable console output
"""
self._idx = 0
self.trials = Trials()
if self._has_maxiteration_field:
if 'solver_max_iterations' not in self.project.__dict__:
msg = "Missing max_iteration entry in project, use default {}!".format(DEFAULTITERATIONS)
LOG.warning(msg)
print("WARNING: {}".format(msg))
setattr(self.project, 'solver_max_iterations', DEFAULTITERATIONS)
self._max_iterations = self.project.solver_max_iterations
start_time = datetime.datetime.now()
try:
search_space = self.convert_searchspace(self.project.hyperparameter)
except Exception as e:
msg = "Failed to convert searchspace, error: {}".format(e)
LOG.error(msg)
raise AssertionError(msg)
try:
self.execute_solver(search_space)
except Exception as e:
msg = "Failed to execute solver, error: {}".format(e)
LOG.error(msg)
raise AssertionError(msg)
end_time = datetime.datetime.now()
dt = end_time - start_time
days = divmod(dt.total_seconds(), 86400)
hours = divmod(days[1], 3600)
minutes = divmod(hours[1], 60)
seconds = divmod(minutes[1], 1)
milliseconds = divmod(seconds[1], 0.001)
self._total_duration = [int(days[0]), int(hours[0]), int(minutes[0]), int(seconds[0]), int(milliseconds[0])]
if print_stats:
self.print_best()
self.print_timestats()
def get_results(self):
"""
This function returns a complete optimization history as pandas DataFrame and a dict with the optimal parameter set.
:return: [DataFrame], [dict] history and optimal parameter set
"""
assert isinstance(self.trials, Trials), "precondition violation, wrong trials type! Maybe solver was not yet executed?"
results = {'duration': [], 'losses': [], 'status': []}
pset = self.trials.trials[0]['misc']['vals']
for p in pset.keys():
results[p] = []
for n, trial in enumerate(self.trials.trials):
t1 = trial['book_time']
t2 = trial['refresh_time']
results['duration'].append((t2 - t1).microseconds / 1000.0)
results['losses'].append(trial['result']['loss'])
results['status'].append(trial['result']['status'] == 'ok')
losses = np.array(results['losses'])
results['losses'] = list(losses)
pset = trial['misc']['vals']
for p in pset.items():
results[p[0]].append(p[1][0])
return pd.DataFrame.from_dict(results), self.best
def print_best(self):
print("\n")
print("#" * 40)
print("### Best Parameter Choice ###")
print("#" * 40)
for name, value in self.best.items():
print(" - {}\t:\t{}".format(name, value))
print("\n - number of iterations\t:\t{}".format(self.trials.trials[-1]['tid']+1))
print(" - total time\t:\t{}d:{}h:{}m:{}s:{}ms".format(self._total_duration[0],
self._total_duration[1],
self._total_duration[2],
self._total_duration[3],
self._total_duration[4]))
print("#" * 40)
def compute_time_statistics(self):
dts = []
for trial in self._trials.trials:
if 'book_time' in trial.keys() and 'refresh_time' in trial.keys():
dt = trial['refresh_time'] - trial['book_time']
dts.append(dt.total_seconds())
self._time_per_iteration = np.mean(dts) * 1e3
self._accumulated_blackbox_time = np.sum(dts) * 1e3
tmp = self.total_duration - self._accumulated_blackbox_time
self._solver_overhead = int(np.round(100.0 / (self.total_duration+1e-12) * tmp))
def print_timestats(self):
print("\n")
print("#" * 40)
print("### Timing Statistics ###")
print("#" * 40)
print(" - per iteration: {}ms".format(int(self.time_per_iteration*1e4)/10000))
print(" - total time: {}d:{}h:{}m:{}s:{}ms".format(self._total_duration[0],
self._total_duration[1],
self._total_duration[2],
self._total_duration[3],
self._total_duration[4]))
print("#" * 40)
print(" - solver overhead: {}%".format(self.solver_overhead))
+ def start_viewer(self, port=8097, server="http://localhost"):
+ try:
+ self._visdom_viewer = VisdomViewer(self._project, port, server)
+ except Exception as e:
+ import warnings
+ warnings.warn("Failed starting VisdomViewer. Is the server running? If not start it via $visdom")
+ LOG.error("Failed starting VisdomViewer: {}".format(e))
+ self._visdom_viewer = None
+
@property
def project(self):
return self._project
@project.setter
def project(self, value):
if not isinstance(value, HyppopyProject):
msg = "Input error, project_manager of type: {} not allowed!".format(type(value))
LOG.error(msg)
raise IOError(msg)
self._project = value
@property
def blackbox(self):
return self._blackbox
@blackbox.setter
def blackbox(self, value):
if isinstance(value, types.FunctionType) or isinstance(value, BlackboxFunction) or isinstance(value, VirtualFunction):
self._blackbox = value
else:
self._blackbox = None
msg = "Input error, blackbox of type: {} not allowed!".format(type(value))
LOG.error(msg)
raise IOError(msg)
@property
def best(self):
return self._best
@best.setter
def best(self, value):
if not isinstance(value, dict):
msg = "Input error, best of type: {} not allowed!".format(type(value))
LOG.error(msg)
raise IOError(msg)
self._best = value
@property
def trials(self):
return self._trials
@trials.setter
def trials(self, value):
self._trials = value
@property
def max_iterations(self):
return self._max_iterations
@max_iterations.setter
def max_iterations(self, value):
if not isinstance(value, int):
msg = "Input error, max_iterations of type: {} not allowed!".format(type(value))
LOG.error(msg)
raise IOError(msg)
if value < 1:
msg = "Precondition violation, max_iterations < 1!"
LOG.error(msg)
raise IOError(msg)
self._max_iterations = value
@property
def total_duration(self):
return (self._total_duration[0]*86400 + self._total_duration[1] * 3600 + self._total_duration[2] * 60 + self._total_duration[3]) * 1000 + self._total_duration[4]
@property
def solver_overhead(self):
if self._solver_overhead is None:
self.compute_time_statistics()
return self._solver_overhead
@property
def time_per_iteration(self):
if self._time_per_iteration is None:
self.compute_time_statistics()
return self._time_per_iteration
@property
def accumulated_blackbox_time(self):
if self._accumulated_blackbox_time is None:
self.compute_time_statistics()
return self._accumulated_blackbox_time
diff --git a/hyppopy/Solver/OptunaSolver.py b/hyppopy/Solver/OptunaSolver.py
index 9074a31..d3d4871 100644
--- a/hyppopy/Solver/OptunaSolver.py
+++ b/hyppopy/Solver/OptunaSolver.py
@@ -1,82 +1,80 @@
# DKFZ
#
#
# Copyright (c) German Cancer Research Center,
# Division of Medical Image Computing.
# All rights reserved.
#
# This software is distributed WITHOUT ANY WARRANTY; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.
#
# See LICENSE
-#
-# Author: Sven Wanner (s.wanner@dkfz.de)
import os
import optuna
import logging
import warnings
import numpy as np
from pprint import pformat
from hyppopy.globals import DEBUGLEVEL
from hyppopy.solver.HyppopySolver import HyppopySolver
LOG = logging.getLogger(os.path.basename(__file__))
LOG.setLevel(DEBUGLEVEL)
class OptunaSolver(HyppopySolver):
def __init__(self, project=None):
HyppopySolver.__init__(self, project)
self._searchspace = None
def reformat_parameter(self, params):
out_params = {}
for name, value in params.items():
if self._searchspace[name]["domain"] == "categorical":
out_params[name] = self._searchspace[name]["data"][int(np.round(value))]
else:
if self._searchspace[name]["type"] == "int":
out_params[name] = int(np.round(value))
else:
out_params[name] = value
return out_params
def trial_cache(self, trial):
params = {}
for name, param in self._searchspace.items():
if param["domain"] == "categorical":
params[name] = trial.suggest_categorical(name, param["data"])
else:
params[name] = trial.suggest_uniform(name, param["data"][0], param["data"][1])
return self.loss_function(**params)
def loss_function_call(self, params):
for key in params.keys():
if self.project.get_typeof(key) is int:
params[key] = int(round(params[key]))
return self.blackbox(**params)
def execute_solver(self, searchspace):
LOG.debug("execute_solver using solution space:\n\n\t{}\n".format(pformat(searchspace)))
self._searchspace = searchspace
try:
study = optuna.create_study()
study.optimize(self.trial_cache, n_trials=self.max_iterations)
self.best = study.best_trial.params
except Exception as e:
LOG.error("internal error in bayes_opt maximize occured. {}".format(e))
raise BrokenPipeError("internal error in bayes_opt maximize occured. {}".format(e))
def convert_searchspace(self, hyperparameter):
LOG.debug("convert input parameter\n\n\t{}\n".format(pformat(hyperparameter)))
for name, param in hyperparameter.items():
if param["domain"] != "categorical" and param["domain"] != "uniform":
msg = "Warning: Optuna cannot handle {} domain. Only uniform and categorical domains are supported!".format(param["domain"])
warnings.warn(msg)
LOG.warning(msg)
return hyperparameter
diff --git a/hyppopy/Solver/OptunitySolver.py b/hyppopy/Solver/OptunitySolver.py
index 7a40117..5a8b337 100644
--- a/hyppopy/Solver/OptunitySolver.py
+++ b/hyppopy/Solver/OptunitySolver.py
@@ -1,98 +1,96 @@
# DKFZ
#
#
# Copyright (c) German Cancer Research Center,
# Division of Medical Image Computing.
# All rights reserved.
#
# This software is distributed WITHOUT ANY WARRANTY; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.
#
# See LICENSE
-#
-# Author: Sven Wanner (s.wanner@dkfz.de)
import os
import logging
import optunity
import warnings
from pprint import pformat
from hyppopy.globals import DEBUGLEVEL
LOG = logging.getLogger(os.path.basename(__file__))
LOG.setLevel(DEBUGLEVEL)
from hyppopy.solver.HyppopySolver import HyppopySolver
class OptunitySolver(HyppopySolver):
def __init__(self, project=None):
HyppopySolver.__init__(self, project)
self._solver_info = None
self.opt_trials = None
def loss_function_call(self, params):
for key in params.keys():
if self.project.get_typeof(key) is int:
params[key] = int(round(params[key]))
return self.blackbox(**params)
def execute_solver(self, searchspace):
LOG.debug("execute_solver using solution space:\n\n\t{}\n".format(pformat(searchspace)))
try:
self.best, _, _ = optunity.minimize_structured(f=self.loss_function,
num_evals=self.max_iterations,
search_space=searchspace)
except Exception as e:
LOG.error("internal error in optunity.minimize_structured occured. {}".format(e))
raise BrokenPipeError("internal error in optunity.minimize_structured occured. {}".format(e))
def split_categorical(self, pdict):
categorical = {}
uniform = {}
for name, pset in pdict.items():
for key, value in pset.items():
if key == 'domain' and value == 'categorical':
categorical[name] = pset
elif key == 'domain':
if value != 'uniform':
msg = "Warning: Optunity cannot handle {} domain. Only uniform and categorical domains are supported!".format(value)
warnings.warn(msg)
LOG.warning(msg)
uniform[name] = pset
return categorical, uniform
def convert_searchspace(self, hyperparameter):
LOG.debug("convert input parameter\n\n\t{}\n".format(pformat(hyperparameter)))
solution_space = {}
# split input in categorical and non-categorical data
cat, uni = self.split_categorical(hyperparameter)
# build up dictionary keeping all non-categorical data
uniforms = {}
for key, value in uni.items():
for key2, value2 in value.items():
if key2 == 'data':
if len(value2) == 3:
uniforms[key] = value2[0:2]
elif len(value2) == 2:
uniforms[key] = value2
else:
raise AssertionError("precondition violation, optunity searchspace needs list with left and right range bounds!")
if len(cat) == 0:
return uniforms
# build nested categorical structure
inner_level = uniforms
for key, value in cat.items():
tmp = {}
tmp2 = {}
for key2, value2 in value.items():
if key2 == 'data':
for elem in value2:
tmp[elem] = inner_level
tmp2[key] = tmp
inner_level = tmp2
solution_space = tmp2
return solution_space
diff --git a/hyppopy/Solver/RandomsearchSolver.py b/hyppopy/Solver/RandomsearchSolver.py
index aa297c3..4938fa5 100644
--- a/hyppopy/Solver/RandomsearchSolver.py
+++ b/hyppopy/Solver/RandomsearchSolver.py
@@ -1,159 +1,157 @@
# DKFZ
#
#
# Copyright (c) German Cancer Research Center,
# Division of Medical Image Computing.
# All rights reserved.
#
# This software is distributed WITHOUT ANY WARRANTY; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.
#
# See LICENSE
-#
-# Author: Sven Wanner (s.wanner@dkfz.de)
import os
import copy
import random
import logging
import numpy as np
from pprint import pformat
from hyppopy.globals import DEBUGLEVEL
from hyppopy.solver.HyppopySolver import HyppopySolver
LOG = logging.getLogger(os.path.basename(__file__))
LOG.setLevel(DEBUGLEVEL)
def draw_uniform_sample(param):
"""
function draws a random sample from a uniform range
:param param: [dict] input hyperparameter discription
:return: random sample value of type data['type']
"""
assert param['type'] != 'str', "cannot sample a string list!"
assert param['data'][0] < param['data'][1], "precondition violation: data[0] > data[1]!"
s = random.random()
s *= np.abs(param['data'][1] - param['data'][0])
s += param['data'][0]
if param['type'] == 'int':
s = int(np.round(s))
if s < param['data'][0]:
s = int(param['data'][0])
if s > param['data'][1]:
s = int(param['data'][1])
return s
def draw_normal_sample(param):
"""
function draws a random sample from a normal distributed range
:param param: [dict] input hyperparameter discription
:return: random sample value of type data['type']
"""
assert param['type'] != 'str', "cannot sample a string list!"
assert param['data'][0] < param['data'][1], "precondition violation: data[0] > data[1]!"
mu = (param['data'][1] - param['data'][0]) / 2
sigma = mu / 3
s = np.random.normal(loc=param['data'][0] + mu, scale=sigma)
if s > param['data'][1]:
s = param['data'][1]
if s < param['data'][0]:
s = param['data'][0]
s = float(s)
if param["type"] == "int":
s = int(np.round(s))
return s
def draw_loguniform_sample(param):
"""
function draws a random sample from a logarithmic distributed range
:param param: [dict] input hyperparameter discription
:return: random sample value of type data['type']
"""
assert param['type'] != 'str', "cannot sample a string list!"
assert param['data'][0] < param['data'][1], "precondition violation: data[0] > data[1]!"
p = copy.deepcopy(param)
p['data'][0] = np.log(param['data'][0])
p['data'][1] = np.log(param['data'][1])
assert p['data'][0] is not np.nan, "Precondition violation, left bound input error, results in nan!"
assert p['data'][1] is not np.nan, "Precondition violation, right bound input error, results in nan!"
x = draw_uniform_sample(p)
s = np.exp(x)
if s > param['data'][1]:
s = param['data'][1]
if s < param['data'][0]:
s = param['data'][0]
return s
def draw_categorical_sample(param):
"""
function draws a random sample from a categorical list
:param param: [dict] input hyperparameter discription
:return: random sample value of type data['type']
"""
return random.sample(param['data'], 1)[0]
def draw_sample(param):
"""
function draws a sample from the input hyperparameter descriptor depending on it's domain
:param param: [dict] input hyperparameter discription
:return: random sample value of type data['type']
"""
assert isinstance(param, dict), "input error, hyperparam descriptors of type {} not allowed!".format(type(param))
assert 'domain' in param.keys(), "input error, hyperparam descriptors need a domain key!"
assert 'data' in param.keys(), "input error, hyperparam descriptors need a data key!"
assert 'type' in param.keys(), "input error, hyperparam descriptors need a type key!"
if param['domain'] == "uniform":
return draw_uniform_sample(param)
elif param['domain'] == "normal":
return draw_normal_sample(param)
elif param['domain'] == "loguniform":
return draw_loguniform_sample(param)
elif param['domain'] == "categorical":
return draw_categorical_sample(param)
else:
raise LookupError("Unknown domain {}".format(param['domain']))
class RandomsearchSolver(HyppopySolver):
"""
The RandomsearchSolver class implements a randomsearch optimization. The randomsearch supports
categorical, uniform, normal and loguniform sampling. The solver draws an independent sample
from the parameter space each iteration."""
def __init__(self, project=None):
HyppopySolver.__init__(self, project)
def loss_function_call(self, params):
loss = self.blackbox(**params)
if loss is None:
return np.nan
return loss
def execute_solver(self, searchspace):
N = self.max_iterations
try:
for n in range(N):
params = {}
for name, p in searchspace.items():
params[name] = draw_sample(p)
self.loss_function(**params)
except Exception as e:
msg = "internal error in randomsearch execute_solver occured. {}".format(e)
LOG.error(msg)
raise BrokenPipeError(msg)
self.best = self._trials.argmin
def convert_searchspace(self, hyperparameter):
"""
this function simply pipes the input parameter through, the sample
drawing functions are responsible for interpreting the parameter.
:param hyperparameter: [dict] hyperparameter space
:return: [dict] hyperparameter space
"""
LOG.debug("convert input parameter\n\n\t{}\n".format(pformat(hyperparameter)))
return hyperparameter
diff --git a/hyppopy/SolverPool.py b/hyppopy/SolverPool.py
index 9eb406f..205eed3 100644
--- a/hyppopy/SolverPool.py
+++ b/hyppopy/SolverPool.py
@@ -1,81 +1,79 @@
# DKFZ
#
#
# Copyright (c) German Cancer Research Center,
# Division of Medical Image Computing.
# All rights reserved.
#
# This software is distributed WITHOUT ANY WARRANTY; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.
#
# See LICENSE
-#
-# Author: Sven Wanner (s.wanner@dkfz.de)
from .Singleton import *
import os
import logging
from hyppopy.HyppopyProject import HyppopyProject
from hyppopy.solver.OptunaSolver import OptunaSolver
from hyppopy.solver.BayesOptSolver import BayesOptSolver
from hyppopy.solver.HyperoptSolver import HyperoptSolver
from hyppopy.solver.OptunitySolver import OptunitySolver
from hyppopy.solver.GridsearchSolver import GridsearchSolver
from hyppopy.solver.RandomsearchSolver import RandomsearchSolver
from hyppopy.globals import DEBUGLEVEL
LOG = logging.getLogger(os.path.basename(__file__))
LOG.setLevel(DEBUGLEVEL)
@singleton_object
class SolverPool(metaclass=Singleton):
def __init__(self):
self._solver_list = ["hyperopt",
"optunity",
"bayesopt",
"optuna",
"randomsearch",
"gridsearch"]
def get_solver_names(self):
return self._solver_list
def get(self, solver_name=None, project=None):
if solver_name is not None:
assert isinstance(solver_name, str), "precondition violation, solver_name type str expected, got {} instead!".format(type(solver_name))
if project is not None:
assert isinstance(project, HyppopyProject), "precondition violation, project type HyppopyProject expected, got {} instead!".format(type(project))
if "custom_use_solver" in project.__dict__:
solver_name = project.custom_use_solver
if solver_name not in self._solver_list:
raise AssertionError("Solver named [{}] not implemented!".format(solver_name))
if solver_name == "hyperopt":
if project is not None:
return HyperoptSolver(project)
return HyperoptSolver()
elif solver_name == "optunity":
if project is not None:
return OptunitySolver(project)
return OptunitySolver()
elif solver_name == "bayesopt":
if project is not None:
return BayesOptSolver(project)
return BayesOptSolver()
elif solver_name == "optuna":
if project is not None:
return OptunaSolver(project)
return OptunaSolver()
elif solver_name == "gridsearch":
if project is not None:
return GridsearchSolver(project)
return GridsearchSolver()
elif solver_name == "randomsearch":
if project is not None:
return RandomsearchSolver(project)
return RandomsearchSolver()
diff --git a/hyppopy/VirtualFunction.py b/hyppopy/VirtualFunction.py
index 67950bc..b7af171 100644
--- a/hyppopy/VirtualFunction.py
+++ b/hyppopy/VirtualFunction.py
@@ -1,225 +1,223 @@
# DKFZ
#
#
# Copyright (c) German Cancer Research Center,
# Division of Medical Image Computing.
# All rights reserved.
#
# This software is distributed WITHOUT ANY WARRANTY; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.
#
# See LICENSE
-#
-# Author: Sven Wanner (s.wanner@dkfz.de)
########################################################################################################################
# USAGE
#
# The class VirtualFunction is meant to be a virtual energy function with an arbitrary dimensionality. The user can
# simply scribble functions as a binary image using e.g. Gimp, defining their ranges using .cfg file and loading them
# into the VirtualFunction. An instance of the class can then be used like a normal function returning the sampling of
# each dimension loaded.
#
# 1. create binary images (IMPORTANT same shape for each), background black the function signature white, ensure that
# each column has a white pixel. If more than one pixel appears in a column, only the lowest will be used.
#
# 2. create a .cfg file, see an example in hyppopy/virtualparameterspace
#
# 3. vfunc = VirtualFunction()
# vfunc.load_images(path/of/your/binaryfiles/and/the/configfile)
#
# 4. use vfunc like a normal function, if you loaded 4 dimension binary images use it like f = vfunc(a,b,c,d)
########################################################################################################################
import os
import sys
import numpy as np
import configparser
from glob import glob
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
from hyppopy.globals import VFUNCDATAPATH
class VirtualFunction(object):
def __init__(self):
self.config = None
self.data = None
self.axis = []
def __call__(self, *args, **kwargs):
if len(kwargs) == self.dims():
args = [0]*len(kwargs)
for key, value in kwargs.items():
index = int(key.split("_")[1])
args[index] = value
assert len(args) == self.dims(), "wrong number of arguments!"
for i in range(len(args)):
assert self.axis[i][0] <= args[i] <= self.axis[i][1], "out of range access on axis {}!".format(i)
lpos, rpos, fracs = self.pos_to_indices(args)
fl = self.data[(list(range(self.dims())), lpos)]
fr = self.data[(list(range(self.dims())), rpos)]
return np.sum(fl*np.array(fracs) + fr*(1-np.array(fracs)))
def clear(self):
self.axis.clear()
self.data = None
self.config = None
def dims(self):
return self.data.shape[0]
def size(self):
return self.data.shape[1]
def range(self, dim):
return np.abs(self.axis[dim][1] - self.axis[dim][0])
def minima(self):
glob_mins = []
for dim in range(self.dims()):
x = []
fmin = np.min(self.data[dim, :])
for _x in range(self.size()):
if self.data[dim, _x] <= fmin:
x.append(_x/self.size()*(self.axis[dim][1]-self.axis[dim][0])+self.axis[dim][0])
glob_mins.append([x, fmin])
return glob_mins
def pos_to_indices(self, positions):
lpos = []
rpos = []
pfracs = []
for n in range(self.dims()):
pos = positions[n]
pos -= self.axis[n][0]
pos /= np.abs(self.axis[n][1]-self.axis[n][0])
pos *= self.data.shape[1]-1
lp = int(np.floor(pos))
if lp < 0:
lp = 0
rp = int(np.ceil(pos))
if rp > self.data.shape[1]-1:
rp = self.data.shape[1]-1
pfracs.append(1.0-(pos-np.floor(pos)))
lpos.append(lp)
rpos.append(rp)
return lpos, rpos, pfracs
def plot(self, dim=None, title=""):
if dim is None:
dim = list(range(self.dims()))
else:
dim = [dim]
fig = plt.figure(figsize=(10, 8))
for i in range(len(dim)):
width = np.abs(self.axis[dim[i]][1]-self.axis[dim[i]][0])
ax = np.arange(self.axis[dim[i]][0], self.axis[dim[i]][1], width/self.size())
plt.plot(ax, self.data[dim[i], :], '.', label='axis_{}'.format(str(dim[i]).zfill(2)))
plt.legend()
plt.grid()
plt.title(title)
plt.show()
def add_dimension(self, data, x_range):
if self.data is None:
self.data = data
if len(self.data.shape) == 1:
self.data = self.data.reshape((1, self.data.shape[0]))
else:
if len(data.shape) == 1:
data = data.reshape((1, data.shape[0]))
assert self.data.shape[1] == data.shape[1], "shape mismatch while adding dimension!"
dims = self.data.shape[0]
size = self.data.shape[1]
tmp = np.append(self.data, data)
self.data = tmp.reshape((dims+1, size))
self.axis.append(x_range)
def load_default(self, name="3D"):
path = os.path.join(VFUNCDATAPATH, "{}".format(name))
if os.path.exists(path):
self.load_images(path)
else:
raise FileExistsError("No virtualfunction of dimension {} available".format(name))
def load_images(self, path):
self.config = None
self.data = None
self.axis.clear()
img_fnames = []
for f in glob(path + os.sep + "*"):
if f.endswith(".png"):
img_fnames.append(f)
elif f.endswith(".cfg"):
self.config = self.read_config(f)
else:
print("WARNING: files of type {} not supported, the file {} is ignored!".format(f.split(".")[-1],
os.path.basename(f)))
if self.config is None:
print("Aborted, failed to read configfile!")
sys.exit()
sections = self.config.sections()
if len(sections) != len(img_fnames):
print("Aborted, inconsistent number of image tmplates and axis specifications!")
sys.exit()
img_fnames.sort()
size_x = None
size_y = None
for n, fname in enumerate(img_fnames):
img = mpimg.imread(fname)
if len(img.shape) > 2:
img = img[:, :, 0]
if size_x is None:
size_x = img.shape[1]
if size_y is None:
size_y = img.shape[0]
self.data = np.zeros((len(img_fnames), size_x), dtype=np.float32)
assert img.shape[0] == size_y, "Shape mismatch in dimension y {} is not {}".format(img.shape[0], size_y)
assert img.shape[1] == size_x, "Shape mismatch in dimension x {} is not {}".format(img.shape[1], size_x)
self.sample_image(img, n)
def sample_image(self, img, dim):
sec_name = "axis_{}".format(str(dim).zfill(2))
assert sec_name in self.config.sections(), "config section {} not found!".format(sec_name)
settings = self.get_axis_settings(sec_name)
self.axis.append([float(settings['min_x']), float(settings['max_x'])])
y_range = [float(settings['min_y']), float(settings['max_y'])]
for x in range(img.shape[1]):
candidates = np.where(img[:, x] > 0)
assert len(candidates[0]) > 0, "non function value in image detected, ensure each column has at least one value > 0!"
y_pos = candidates[0][0]/img.shape[0]
self.data[dim, x] = 1-y_pos
self.data[dim, :] *= np.abs(y_range[1] - y_range[0])
self.data[dim, :] += y_range[0]
def read_config(self, fname):
try:
config = configparser.ConfigParser()
config.read(fname)
return config
except Exception as e:
print(e)
return None
def get_axis_settings(self, section):
dict1 = {}
options = self.config.options(section)
for option in options:
try:
dict1[option] = self.config.get(section, option)
if dict1[option] == -1:
print("skip: %s" % option)
except:
print("exception on %s!" % option)
dict1[option] = None
return dict1
diff --git a/hyppopy/VisdomViewer.py b/hyppopy/VisdomViewer.py
new file mode 100644
index 0000000..bff9a94
--- /dev/null
+++ b/hyppopy/VisdomViewer.py
@@ -0,0 +1,114 @@
+import warnings
+import numpy as np
+from visdom import Visdom
+import matplotlib.pyplot as plt
+
+
+def time_formatter(time_s):
+ if time_s < 0.01:
+ return int(time_s * 1000.0 * 1000) / 1000.0, "ms"
+ elif 100 < time_s < 3600:
+ return int(time_s / 60 * 1000) / 1000.0, "min"
+ elif time_s >= 3600:
+ return int(time_s / 3600 * 1000) / 1000.0, "h"
+ else:
+ return int(time_s * 1000) / 1000.0, "s"
+
+
+class VisdomViewer(object):
+
+ def __init__(self, project, port=8097, server="http://localhost"):
+ self._viz = Visdom(port=port, server=server)
+ self._enabled = self._viz.check_connection(timeout_seconds=3)
+ if not self._enabled:
+ warnings.warn("No connection to visdom server established. Visualization cannot be displayed!")
+
+ self._project = project
+ self._best_win = None
+ self._best_loss = None
+ self._loss_iter_plot = None
+ self._status_report = None
+ self._axis_tags = None
+ self._axis_plots = None
+
+ def plot_losshistory(self, input_data):
+ loss = np.array([input_data["loss"]])
+ iter = np.array([input_data["iterations"]])
+ if self._loss_iter_plot is None:
+ self._loss_iter_plot = self._viz.line(loss, X=iter, opts=dict(
+ markers=True,
+ markersize=5,
+ dash=np.array(['dashdot']),
+ title="Loss History",
+ xlabel='iteration',
+ ylabel='loss'
+ ))
+ else:
+ self._viz.line(loss, X=iter, win=self._loss_iter_plot, update='append')
+
+ def plot_hyperparameter(self, input_data):
+ if self._axis_plots is None:
+ self._axis_tags = []
+ self._axis_plots = {}
+ for item in input_data.keys():
+ if item == "refresh_time" or item == "book_time" or item == "iterations" or item == "status" or item == "loss":
+ continue
+ self._axis_tags.append(item)
+ for axis in self._axis_tags:
+ xlabel = "value"
+ if isinstance(input_data[axis], str):
+ if self._project.hyperparameter[axis]["domain"] == "categorical":
+ xlabel = '-'.join(self._project.hyperparameter[axis]["data"])
+ input_data[axis] = self._project.hyperparameter[axis]["data"].index(input_data[axis])
+ axis_loss = np.array([input_data[axis], input_data["loss"]]).reshape(1, -1)
+ self._axis_plots[axis] = self._viz.scatter(axis_loss, opts=dict(
+ markersize=5,
+ title=axis,
+ xlabel=xlabel,
+ ylabel='loss'))
+ else:
+ for axis in self._axis_tags:
+ if isinstance(input_data[axis], str):
+ if self._project.hyperparameter[axis]["domain"] == "categorical":
+ input_data[axis] = self._project.hyperparameter[axis]["data"].index(input_data[axis])
+ axis_loss = np.array([input_data[axis], input_data["loss"]]).reshape(1, -1)
+ self._viz.scatter(axis_loss, win=self._axis_plots[axis], update='append')
+
+ def show_statusreport(self, input_data):
+ duration = input_data['refresh_time'] - input_data['book_time']
+ duration, time_format = time_formatter(duration.total_seconds())
+ report = "Iteration {}: {}{} -> {}\n".format(input_data["iterations"], duration, time_format, input_data["status"])
+ if self._status_report is None:
+ self._status_report = self._viz.text(report)
+ else:
+ self._viz.text(report, win=self._status_report, append=True)
+
+ def show_best(self, input_data):
+ if self._best_win is None:
+ self._best_loss = input_data["loss"]
+ txt = "Best Parameter Set: