diff --git a/hyppopy/Solver/OptunitySolver.py b/hyppopy/Solver/OptunitySolver.py index 359e930..fcd000b 100644 --- a/hyppopy/Solver/OptunitySolver.py +++ b/hyppopy/Solver/OptunitySolver.py @@ -1,118 +1,123 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical and Biological Informatics. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE.txt or http://www.mitk.org for details. # # Author: Sven Wanner (s.wanner@dkfz.de) import os import copy import logging import optunity import datetime import numpy as np from pprint import pformat from hyperopt import Trials from hyppopy.globals import DEBUGLEVEL LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) from .HyppopySolver import HyppopySolver from ..helpers import split_categorical from ..BlackboxFunction import BlackboxFunction class OptunitySolver(HyppopySolver): def __init__(self, project=None): HyppopySolver.__init__(self, project) self._solver_info = None self.opt_trials = None self._idx = None def loss_function(self, **params): self._idx += 1 vals = {} idx = {} for key, value in params.items(): vals[key] = [value] idx[key] = [self._idx] trial = {'tid': self._idx, 'result': {'loss': None, 'status': 'ok'}, 'misc': { 'tid': self._idx, 'idxs': idx, 'vals': vals }, 'book_time': datetime.datetime.now(), 'refresh_time': None } try: for key in params.keys(): if self.project.get_typeof(key) is int: params[key] = int(round(params[key])) loss = self.blackbox(**params) trial['result']['loss'] = loss trial['result']['status'] = 'ok' except Exception as e: LOG.error("computing loss failed due to:\n {}".format(e)) loss = np.nan trial['result']['loss'] = np.nan trial['result']['status'] = 'failed' trial['refresh_time'] = datetime.datetime.now() self._trials.trials.append(trial) if isinstance(self.blackbox, BlackboxFunction) and self.blackbox.callback_func is not None: cbd = copy.deepcopy(params) cbd['iterations'] = self._idx cbd['loss'] = loss cbd['status'] = trial['result']['status'] self.blackbox.callback_func(**cbd) return loss def execute_solver(self, searchspace): LOG.debug("execute_solver using solution space:\n\n\t{}\n".format(pformat(searchspace))) self.trials = Trials() self._idx = 0 try: self.best, self.opt_trials, self._solver_info = optunity.minimize_structured(f=self.loss_function, num_evals=self.max_iterations, search_space=searchspace) except Exception as e: LOG.error("internal error in optunity.minimize_structured occured. {}".format(e)) raise BrokenPipeError("internal error in optunity.minimize_structured occured. {}".format(e)) def convert_searchspace(self, hyperparameter): solution_space = {} # split input in categorical and non-categorical data cat, uni = split_categorical(hyperparameter) # build up dictionary keeping all non-categorical data uniforms = {} for key, value in uni.items(): for key2, value2 in value.items(): if key2 == 'data': - uniforms[key] = value2 + if len(value2) == 3: + uniforms[key] = value2[0:2] + elif len(value2) == 2: + uniforms[key] = value2 + else: + raise AssertionError("precondition violation, optunity searchspace needs list with left and right range bounds!") if len(cat) == 0: return uniforms # build nested categorical structure inner_level = uniforms for key, value in cat.items(): tmp = {} tmp2 = {} for key2, value2 in value.items(): if key2 == 'data': for elem in value2: tmp[elem] = inner_level tmp2[key] = tmp inner_level = tmp2 solution_space = tmp2 return solution_space diff --git a/hyppopy/tests/test_hyperoptsolver.py b/hyppopy/tests/test_hyperoptsolver.py new file mode 100644 index 0000000..6453efc --- /dev/null +++ b/hyppopy/tests/test_hyperoptsolver.py @@ -0,0 +1,66 @@ +# DKFZ +# +# +# Copyright (c) German Cancer Research Center, +# Division of Medical and Biological Informatics. +# All rights reserved. +# +# This software is distributed WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. +# +# See LICENSE.txt or http://www.mitk.org for details. +# +# Author: Sven Wanner (s.wanner@dkfz.de) + +import unittest +import matplotlib.pylab as plt + +from ..solver.HyperoptSolver import * +from ..VirtualFunction import VirtualFunction +from hyppopy.HyppopyProject import HyppopyProject + + +class HyperoptSolverTestSuite(unittest.TestCase): + + def setUp(self): + pass + + def test_solver_complete(self): + config = { + "hyperparameter": { + "axis_00": { + "domain": "normal", + "data": [300, 800], + "type": "float" + }, + "axis_01": { + "domain": "normal", + "data": [-1, 1], + "type": "float" + }, + "axis_02": { + "domain": "uniform", + "data": [0, 10], + "type": "float" + } + }, + "settings": { + "solver": {"max_iterations": 800}, + "custom": {} + }} + + project = HyppopyProject(config) + solver = HyperoptSolver(project) + vfunc = VirtualFunction() + vfunc.load_default() + solver.blackbox = vfunc + solver.run(print_stats=False) + df, best = solver.get_results() + self.assertTrue(570 < best['axis_00'] < 590) + self.assertTrue(0.1 < best['axis_01'] < 0.8) + self.assertTrue(4.5 < best['axis_02'] < 6) + + +if __name__ == '__main__': + unittest.main() diff --git a/hyppopy/tests/test_optunitysolver.py b/hyppopy/tests/test_optunitysolver.py new file mode 100644 index 0000000..ebc03e3 --- /dev/null +++ b/hyppopy/tests/test_optunitysolver.py @@ -0,0 +1,66 @@ +# DKFZ +# +# +# Copyright (c) German Cancer Research Center, +# Division of Medical and Biological Informatics. +# All rights reserved. +# +# This software is distributed WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. +# +# See LICENSE.txt or http://www.mitk.org for details. +# +# Author: Sven Wanner (s.wanner@dkfz.de) + +import unittest +import matplotlib.pylab as plt + +from ..solver.OptunitySolver import * +from ..VirtualFunction import VirtualFunction +from hyppopy.HyppopyProject import HyppopyProject + + +class OptunitySolverTestSuite(unittest.TestCase): + + def setUp(self): + pass + + def test_solver_complete(self): + config = { + "hyperparameter": { + "axis_00": { + "domain": "normal", + "data": [300, 800], + "type": "float" + }, + "axis_01": { + "domain": "normal", + "data": [-1, 1], + "type": "float" + }, + "axis_02": { + "domain": "uniform", + "data": [0, 10], + "type": "float" + } + }, + "settings": { + "solver": {"max_iterations": 800}, + "custom": {} + }} + + project = HyppopyProject(config) + solver = OptunitySolver(project) + vfunc = VirtualFunction() + vfunc.load_default() + solver.blackbox = vfunc + solver.run(print_stats=False) + df, best = solver.get_results() + self.assertTrue(570 < best['axis_00'] < 590) + self.assertTrue(0.1 < best['axis_01'] < 0.8) + self.assertTrue(4.5 < best['axis_02'] < 6) + + +if __name__ == '__main__': + unittest.main() diff --git a/hyppopy/tests/test_randomsearchsolver.py b/hyppopy/tests/test_randomsearchsolver.py index 8ef60f8..4bb5c54 100644 --- a/hyppopy/tests/test_randomsearchsolver.py +++ b/hyppopy/tests/test_randomsearchsolver.py @@ -1,134 +1,131 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical and Biological Informatics. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE.txt or http://www.mitk.org for details. # # Author: Sven Wanner (s.wanner@dkfz.de) import unittest import matplotlib.pylab as plt from ..solver.RandomsearchSolver import * from ..VirtualFunction import VirtualFunction from hyppopy.HyppopyProject import HyppopyProject class RandomsearchTestSuite(unittest.TestCase): def setUp(self): pass def test_draw_uniform_sample(self): param = {"data": [0, 1, 10], "type": "float"} values = [] for i in range(10000): values.append(draw_uniform_sample(param)) self.assertTrue(0 <= values[-1] <= 1) self.assertTrue(isinstance(values[-1], float)) hist = plt.hist(values, bins=10, normed=True) std = np.std(hist[0]) mean = np.mean(hist[0]) self.assertTrue(std < 0.05) self.assertTrue(0.9 < mean < 1.1) param = {"data": [0, 10, 11], "type": "int"} values = [] for i in range(10000): values.append(draw_uniform_sample(param)) self.assertTrue(0 <= values[-1] <= 10) self.assertTrue(isinstance(values[-1], int)) hist = plt.hist(values, bins=11, normed=True) std = np.std(hist[0]) mean = np.mean(hist[0]) self.assertTrue(std < 0.05) self.assertTrue(0.09 < mean < 0.11) def test_draw_normal_sample(self): param = {"data": [0, 10, 11], "type": "int"} values = [] for i in range(10000): values.append(draw_normal_sample(param)) self.assertTrue(0 <= values[-1] <= 10) self.assertTrue(isinstance(values[-1], int)) hist = plt.hist(values, bins=11, normed=True) for i in range(1, 5): self.assertTrue(hist[0][i-1]-hist[0][i] < 0) for i in range(5, 10): self.assertTrue(hist[0][i] - hist[0][i+1] > 0) def test_draw_loguniform_sample(self): param = {"data": [1, 1000, 11], "type": "float"} values = [] for i in range(10000): values.append(draw_loguniform_sample(param)) self.assertTrue(1 <= values[-1] <= 1000) self.assertTrue(isinstance(values[-1], float)) hist = plt.hist(values, bins=11, normed=True) for i in range(10): self.assertTrue(hist[0][i] > hist[0][i+1]) def test_draw_categorical_sample(self): param = {"data": [1, 2, 3], "type": int} values = [] for i in range(10000): values.append(draw_categorical_sample(param)) self.assertTrue(values[-1] == 1 or values[-1] == 2 or values[-1] == 3) self.assertTrue(isinstance(values[-1], int)) hist = plt.hist(values, bins=3, normed=True) for i in range(3): self.assertTrue(0.45 < hist[0][i] < 0.55) def test_solver_complete(self): config = { "hyperparameter": { "axis_00": { "domain": "normal", - "data": [300, 800, 11], + "data": [300, 800], "type": "float" }, "axis_01": { "domain": "normal", - "data": [-1, 1, 11], + "data": [-1, 1], "type": "float" }, "axis_02": { "domain": "uniform", - "data": [0, 10, 11], + "data": [0, 10], "type": "float" } }, "settings": { "solver": {"max_iterations": 5000}, "custom": {} }} project = HyppopyProject(config) solver = RandomsearchSolver(project) vfunc = VirtualFunction() vfunc.load_default() solver.blackbox = vfunc solver.run(print_stats=False) df, best = solver.get_results() - print("best['axis_00']={}".format(best['axis_00'])) - print("best['axis_01']={}".format(best['axis_01'])) - print("best['axis_02']={}".format(best['axis_02'])) self.assertTrue(570 < best['axis_00'] < 590) self.assertTrue(0.1 < best['axis_01'] < 0.8) self.assertTrue(4.5 < best['axis_02'] < 6) if __name__ == '__main__': unittest.main()