diff --git a/doc/developer_guide.rst b/doc/developer_guide.rst index af19150..8cf5825 100644 --- a/doc/developer_guide.rst +++ b/doc/developer_guide.rst @@ -1,169 +1,175 @@ **************** Developers Guide **************** The main classes and their connections ************************************** The picture below depicts the releationships between the most important classes of hyppopy. .. image:: _static/class_diagram.png To understand the concept behind Hyppopy the following classes are important: - :py:mod:`hyppopy.solvers.HyppopySolver` - :py:mod:`hyppopy.HyppopyProject` - :py:mod:`hyppopy.BlackboxFunction` The :py:mod:`hyppopy.solvers.HyppopySolver` class is the parent class of all solvers in Hyppopy. It defines an abstract interface that needs to be implemented by each custom solver class. The main idea is to define a common interface for the different approaches the solver libraries are based on. When designing Hyppopy there were three main challenges that drove the design. Each solver library has a different approach to define or describe the hyperparameter space, has a different approach to track the solver information and is different in setting the blackbox function and running the optimization process. To deal with those differences the :py:mod:`hyppopy.solvers.HyppopySolver` class defines the abstract interface functions `convert_searchspace`, `execute_solver`, `loss_function_call` and `define_interface`. Those serve as abstraction layer to handle the individual needs of each solver library. Each solver needs a :py:mod:`hyppopy.HyppopyProject` instance keeping the user configuration input and a :py:mod:`hyppopy.BlackboxFunction` instance, implementing the loss function. Implementing a custom solver **************************** Adding a new solver is only about deriving a new class from :py:mod:`hyppopy.solvers.HyppopySolver` as well as telling the :py:mod:`hyppopy.SolverPool` that it exists. We go through the whole process on the example of the solver :py:mod:`hyppopy.solvers.OptunitySolver`: .. code-block:: python import os import optunity from pprint import pformat from hyppopy.solvers.HyppopySolver import HyppopySolver class OptunitySolver(HyppopySolver): def __init__(self, project=None): HyppopySolver.__init__(self, project) First step is to derive from the HyppopySolver class. Good practice would be that the project can be set via __init__ -and if, is piped through to the HyppopySolver.__init__. Next step is implementing the abstract interface methods. -We start with define_interface. This functions purpose is to define the relevant input parameter and the signature -of the hyperparameter space. Our solver needs an parameter called max_iterations of type int. The hyperparameter -space has a domain that allows values 'uniform' and 'categorical', a field data of type list and a field type of type -type. This guarantees that exceptions are thrown if the user disrespects this signature or forgets to set max_iterations. +and if, is piped through to the HyppopySolver.__init__. + +Next step is implementing the abstract interface methods. We start with define_interface. This functions purpose is to +define the relevant input parameter and the signature of a hyperparameter description. This means the solver developer +can define what parameter the solver expects as well as how a single hyperparameter must be described. The rules defined +here are automatically applied when the solver run method is called and exceptions are thrown if there is a mismatch +between these rules and the settings the user sets via it's config. + +Our solver in this example needs an parameter called max_iterations of type int. The hyperparameter space has a domain +that allows values 'uniform' and 'categorical', a field data of type list and a field type of type type. This guarantees +that exceptions are thrown if the user disrespects this signature or forgets to set max_iterations. .. code-block:: python def define_interface(self): self._add_member("max_iterations", int) self._add_hyperparameter_signature(name="domain", dtype=str, options=["uniform", "categorical"]) self._add_hyperparameter_signature(name="data", dtype=list) self._add_hyperparameter_signature(name="type", dtype=type) Next abstract method to implement is convert_searchspace. This method is responsible for interpreting the users hyperparameter input and convert it to a form the solver framework needs. An input for example can be: .. code-block:: python hyperparameter = { 'C': {'domain': 'uniform', 'data': [0.0001, 20], 'type': float}, 'gamma': {'domain': 'uniform', 'data': [0.0001, 20.0], 'type': float}, 'kernel': {'domain': 'categorical', 'data': ['linear', 'sigmoid', 'poly', 'rbf'], 'type': str}, 'decision_function_shape': {'domain': 'categorical', 'data': ['ovo', 'ovr'], 'type': str'} } Optunity instead expects a hyperparameter space formulation as follows: .. code-block:: python optunity_space = {'decision_function_shape': {'ovo': { 'kernel': { 'linear': {'C': [0.0001, 20], 'gamma': [0.0001, 20.0]}, 'sigmoid': {'C': [0.0001, 20], 'gamma': [0.0001, 20.0]}, 'poly': {'C': [0.0001, 20], 'gamma': [0.0001, 20.0]}, 'rbf': {'C': [0.0001, 20], 'gamma': [0.0001, 20.0]}} }, 'ovr': { 'kernel': { 'linear': {'C': [0.0001, 20], 'gamma': [0.0001, 20.0]}, 'sigmoid': {'C': [0.0001, 20], 'gamma': [0.0001, 20.0]}, 'poly': {'C': [0.0001, 20], 'gamma': [0.0001, 20.0]}, 'rbf': {'C': [0.0001, 20], 'gamma': [0.0001, 20.0]}} } }} This conversion is what convert_searchspace is meant for. .. code-block:: python def convert_searchspace(self, hyperparameter): LOG.debug("convert input parameter\n\n\t{}\n".format(pformat(hyperparameter))) # split input in categorical and non-categorical data cat, uni = self.split_categorical(hyperparameter) # build up dictionary keeping all non-categorical data uniforms = {} for key, value in uni.items(): for key2, value2 in value.items(): if key2 == 'data': if len(value2) == 3: uniforms[key] = value2[0:2] elif len(value2) == 2: uniforms[key] = value2 else: raise AssertionError("precondition violation, optunity searchspace needs list with left and right range bounds!") if len(cat) == 0: return uniforms # build nested categorical structure inner_level = uniforms for key, value in cat.items(): tmp = {} optunity_space = {} for key2, value2 in value.items(): if key2 == 'data': for elem in value2: tmp[elem] = inner_level optunity_space[key] = tmp inner_level = optunity_space return optunity_space Now we have defined how the solver looks from outside and how to convert the parameterspace coming in, we can define how the blackbox function is called. The abstract method loss_function_call is a wrapper function enabling to customize the call of the blackbox function. In case of Optunity we only check if a parameter is of type int and convert it to ensure that no exception are thrown in case of integers are expected in the blackbox. .. code-block:: python def loss_function_call(self, params): for key in params.keys(): if self.project.get_typeof(key) is int: params[key] = int(round(params[key])) return self.blackbox(**params) In execute_solver the actual wrapping of the solver framework call is done. Here call the Optunity optimizing function. A dictionary keeping the optimal parameter set must assigned to self.best. .. code-block:: python def execute_solver(self, searchspace): LOG.debug("execute_solver using solution space:\n\n\t{}\n".format(pformat(searchspace))) try: self.best, _, _ = optunity.minimize_structured(f=self.loss_function, num_evals=self.max_iterations, search_space=searchspace) except Exception as e: LOG.error("internal error in optunity.minimize_structured occured. {}".format(e)) raise BrokenPipeError("internal error in optunity.minimize_structured occured. {}".format(e)) diff --git a/hyppopy/solvers/HyperoptSolver.py b/hyppopy/solvers/HyperoptSolver.py index 673ce8e..7540c71 100644 --- a/hyppopy/solvers/HyperoptSolver.py +++ b/hyppopy/solvers/HyperoptSolver.py @@ -1,202 +1,202 @@ # Hyppopy - A Hyper-Parameter Optimization Toolbox # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE import os import copy import logging import numpy as np from pprint import pformat from hyperopt import fmin, tpe, hp, STATUS_OK, STATUS_FAIL, Trials from hyppopy.globals import DEBUGLEVEL from hyppopy.solvers.HyppopySolver import HyppopySolver from hyppopy.BlackboxFunction import BlackboxFunction LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) class HyperoptSolver(HyppopySolver): def __init__(self, project=None): """ The constructor accepts a HyppopyProject. :param project: [HyppopyProject] project instance, default=None """ HyppopySolver.__init__(self, project) self._searchspace = None def define_interface(self): """ This function is called when HyppopySolver.__init__ function finished. Child classes need to define their individual parameter here by calling the _add_member function for each class member variable need to be defined. Using _add_hyperparameter_signature the structure of a hyperparameter the solver expects must be defined. Both, members and hyperparameter signatures are later get checked, before executing the solver, ensuring settings passed fullfill solver needs. """ self._add_member("max_iterations", int) self._add_hyperparameter_signature(name="domain", dtype=str, options=["uniform", "normal", "loguniform", "categorical"]) self._add_hyperparameter_signature(name="data", dtype=list) self._add_hyperparameter_signature(name="type", dtype=type) def loss_function(self, params): """ Loss function wrapper function. :param params: [dict] hyperparameter set :return: [float] loss """ for name, p in self._searchspace.items(): if p["domain"] != "categorical": if params[name] < p["data"][0]: params[name] = p["data"][0] if params[name] > p["data"][1]: params[name] = p["data"][1] status = STATUS_FAIL try: loss = self.blackbox(**params) if loss is not None: status = STATUS_OK else: loss = 1e9 except Exception as e: LOG.error("execution of self.blackbox(**params) failed due to:\n {}".format(e)) status = STATUS_FAIL loss = 1e9 cbd = copy.deepcopy(params) cbd['iterations'] = self._trials.trials[-1]['tid'] + 1 cbd['loss'] = loss cbd['status'] = status cbd['book_time'] = self._trials.trials[-1]['book_time'] cbd['refresh_time'] = self._trials.trials[-1]['refresh_time'] if isinstance(self.blackbox, BlackboxFunction) and self.blackbox.callback_func is not None: self.blackbox.callback_func(**cbd) if self._visdom_viewer is not None: self._visdom_viewer.update(cbd) return {'loss': loss, 'status': status} def execute_solver(self, searchspace): """ This function is called immediately after convert_searchspace and get the output of the latter as input. It's purpose is to call the solver libs main optimization function. :param searchspace: converted hyperparameter space """ LOG.debug("execute_solver using solution space:\n\n\t{}\n".format(pformat(searchspace))) self.trials = Trials() try: self.best = fmin(fn=self.loss_function, space=searchspace, algo=tpe.suggest, max_evals=self.max_iterations, trials=self.trials) except Exception as e: msg = "internal error in hyperopt.fmin occured. {}".format(e) LOG.error(msg) raise BrokenPipeError(msg) def convert_searchspace(self, hyperparameter): """ This function gets the unified hyppopy-like parameterspace description as input and, if necessary, should convert it into a solver lib specific format. The function is invoked when run is called and what it returns is passed as searchspace argument to the function execute_solver. :param hyperparameter: [dict] nested parameter description dict e.g. {'name': {'domain':'uniform', 'data':[0,1], 'type':'float'}, ...} :return: [object] converted hyperparameter space """ self._searchspace = hyperparameter solution_space = {} for name, content in hyperparameter.items(): param_settings = {'name': name} for key, value in content.items(): if key == 'domain': param_settings['domain'] = value elif key == 'data': param_settings['data'] = value elif key == 'type': param_settings['dtype'] = value solution_space[name] = self.convert(param_settings) return solution_space def convert(self, param_settings): """ Convert searchspace to hyperopt specific searchspace :param param_settings: [dict] hyperparameter description :return: [object] hyperopt description """ name = param_settings["name"] domain = param_settings["domain"] dtype = param_settings["dtype"] data = param_settings["data"] if domain == "uniform": if dtype is float: return hp.uniform(name, data[0], data[1]) elif dtype is int: data = list(np.arange(int(data[0]), int(data[1] + 1))) return hp.choice(name, data) else: msg = "cannot convert the type {} in domain {}".format(dtype, domain) LOG.error(msg) raise LookupError(msg) elif domain == "loguniform": if dtype is float: if data[0] == 0: data[0] += 1e-23 assert data[0] > 0, "precondition Violation, a < 0!" assert data[0] < data[1], "precondition Violation, a > b!" assert data[1] > 0, "precondition Violation, b < 0!" lexp = np.log(data[0]) rexp = np.log(data[1]) assert lexp is not np.nan, "precondition violation, left bound input error, results in nan!" assert rexp is not np.nan, "precondition violation, right bound input error, results in nan!" return hp.loguniform(name, lexp, rexp) else: msg = "cannot convert the type {} in domain {}".format(dtype, domain) LOG.error(msg) raise LookupError(msg) elif domain == "normal": if dtype is float: mu = (data[1] - data[0]) / 2.0 sigma = mu / 3 return hp.normal(name, data[0] + mu, sigma) else: msg = "cannot convert the type {} in domain {}".format(dtype, domain) LOG.error(msg) raise LookupError(msg) elif domain == "categorical": if dtype is str: return hp.choice(name, data) elif dtype is bool: - data = [] + conv = [] for elem in data: - if elem == "true" or elem == "True" or elem == 1 or elem == "1": - data.append(True) - elif elem == "false" or elem == "False" or elem == 0 or elem == "0": - data.append(False) + if elem == "true" or elem == "True" or elem == 1 or elem == "1" or elem == True: + conv.append(True) + elif elem == "false" or elem == "False" or elem == 0 or elem == "0" or elem == False: + conv.append(False) else: msg = "cannot convert the type {} in domain {}, unknown bool type value".format(dtype, domain) LOG.error(msg) raise LookupError(msg) - return hp.choice(name, data) + return hp.choice(name, conv) else: msg = "Precondition violation, domain named {} not available!".format(domain) LOG.error(msg) raise IOError(msg) diff --git a/hyppopy/tests/test_hyperoptsolver.py b/hyppopy/tests/test_hyperoptsolver.py index 8552450..a44bb68 100644 --- a/hyppopy/tests/test_hyperoptsolver.py +++ b/hyppopy/tests/test_hyperoptsolver.py @@ -1,103 +1,188 @@ # Hyppopy - A Hyper-Parameter Optimization Toolbox # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE import unittest from hyppopy.solvers.HyperoptSolver import * from hyppopy.FunctionSimulator import FunctionSimulator from hyppopy.HyppopyProject import HyppopyProject class HyperoptSolverTestSuite(unittest.TestCase): def setUp(self): pass def test_solver_complete(self): config = { "hyperparameter": { "axis_00": { "domain": "uniform", "data": [300, 700], "type": float }, "axis_01": { "domain": "uniform", "data": [0, 0.8], "type": float }, "axis_02": { "domain": "uniform", "data": [3.5, 6.5], "type": float } }, "max_iterations": 500 } project = HyppopyProject(config) solver = HyperoptSolver(project) vfunc = FunctionSimulator() vfunc.load_default() solver.blackbox = vfunc solver.run(print_stats=False) df, best = solver.get_results() self.assertTrue(575 <= best['axis_00'] <= 585) self.assertTrue(0.1 <= best['axis_01'] <= 0.8) self.assertTrue(4.7 <= best['axis_02'] <= 5.3) for status in df['status']: self.assertTrue(status) for loss in df['losses']: self.assertTrue(isinstance(loss, float)) def test_solver_normal(self): config = { "hyperparameter": { "axis_00": { "domain": "normal", "data": [500, 650], "type": float }, "axis_01": { "domain": "normal", "data": [0.1, 0.8], "type": float }, "axis_02": { "domain": "normal", "data": [4.5, 5.5], "type": float } }, "max_iterations": 500, } project = HyppopyProject(config) solver = HyperoptSolver(project) vfunc = FunctionSimulator() vfunc.load_default() solver.blackbox = vfunc solver.run(print_stats=False) df, best = solver.get_results() self.assertTrue(575 <= best['axis_00'] <= 585) self.assertTrue(0.1 <= best['axis_01'] <= 0.8) self.assertTrue(4.7 <= best['axis_02'] <= 5.3) for status in df['status']: self.assertTrue(status) for loss in df['losses']: self.assertTrue(isinstance(loss, float)) + def test_solver_loguniform(self): + config = { + "hyperparameter": { + "axis_00": { + "domain": "normal", + "data": [500, 650], + "type": float + }, + "axis_01": { + "domain": "loguniform", + "data": [0.001, 1], + "type": float + }, + "axis_02": { + "domain": "normal", + "data": [4.5, 5.5], + "type": float + } + }, + "max_iterations": 500, + } + + project = HyppopyProject(config) + solver = HyperoptSolver(project) + vfunc = FunctionSimulator() + vfunc.load_default() + solver.blackbox = vfunc + solver.run(print_stats=False) + df, best = solver.get_results() + self.assertTrue(575 <= best['axis_00'] <= 585) + self.assertTrue(0.001 <= best['axis_01'] <= 1.0) + self.assertTrue(4.7 <= best['axis_02'] <= 5.3) + + for status in df['status']: + self.assertTrue(status) + for loss in df['losses']: + self.assertTrue(isinstance(loss, float)) + + def test_categorical(self): + config = { + "hyperparameter": { + "C": { + "domain": "uniform", + "data": [1, 20], + "type": int + }, + "gamma": { + "domain": "loguniform", + "data": [0.0, 20.0], + "type": float + }, + "kernel": { + "domain": "categorical", + "data": ["linear", "sigmoid", "poly", "rbf"], + "type": str + }, + "with_ovr": { + "domain": "categorical", + "data": [True, False], + "type": bool + } + }, + "max_iterations": 300 + } + project = HyppopyProject(config=config) + from sklearn.svm import SVC + from sklearn.datasets import load_iris + from sklearn.model_selection import cross_val_score + iris_data = load_iris() + data = [iris_data.data, iris_data.target] + + def my_loss_function(data, params): + if params["with_ovr"]: + params["decision_function_shape"] = "ovr" + else: + params["decision_function_shape"] = "ovo" + del params["with_ovr"] + clf = SVC(**params) + return -cross_val_score(estimator=clf, X=data[0], y=data[1], cv=3).mean() + + blackbox = BlackboxFunction(blackbox_func=my_loss_function, data=data) + solver = HyperoptSolver(project=project) + solver.blackbox = blackbox + solver.run() + if __name__ == '__main__': unittest.main() diff --git a/hyppopy/tests/test_solverpool.py b/hyppopy/tests/test_solverpool.py new file mode 100644 index 0000000..ff6b6df --- /dev/null +++ b/hyppopy/tests/test_solverpool.py @@ -0,0 +1,279 @@ +# Hyppopy - A Hyper-Parameter Optimization Toolbox +# +# Copyright (c) German Cancer Research Center, +# Division of Medical Image Computing. +# All rights reserved. +# +# This software is distributed WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. +# +# See LICENSE + +import unittest + +from hyppopy.SolverPool import SolverPool +from hyppopy.HyppopyProject import HyppopyProject +from hyppopy.FunctionSimulator import FunctionSimulator +from hyppopy.solvers.HyperoptSolver import HyperoptSolver +from hyppopy.solvers.OptunitySolver import OptunitySolver +from hyppopy.solvers.OptunaSolver import OptunaSolver +from hyppopy.solvers.RandomsearchSolver import RandomsearchSolver +from hyppopy.solvers.QuasiRandomsearchSolver import QuasiRandomsearchSolver +from hyppopy.solvers.GridsearchSolver import GridsearchSolver + + +class SolverPoolTestSuite(unittest.TestCase): + + def setUp(self): + pass + + def test_PoolContent(self): + names = SolverPool.get_solver_names() + self.assertTrue("hyperopt" in names) + self.assertTrue("optunity" in names) + self.assertTrue("optuna" in names) + self.assertTrue("randomsearch" in names) + self.assertTrue("quasirandomsearch" in names) + self.assertTrue("gridsearch" in names) + + def test_getHyperoptSolver(self): + config = { + "hyperparameter": { + "axis_00": { + "domain": "uniform", + "data": [300, 700], + "type": float + }, + "axis_01": { + "domain": "uniform", + "data": [0, 0.8], + "type": float + }, + "axis_02": { + "domain": "uniform", + "data": [3.5, 6.5], + "type": float + } + }, + "max_iterations": 500 + } + + project = HyppopyProject(config) + solver = SolverPool.get("hyperopt", project) + self.assertTrue(isinstance(solver, HyperoptSolver)) + vfunc = FunctionSimulator() + vfunc.load_default() + solver.blackbox = vfunc + solver.run(print_stats=False) + df, best = solver.get_results() + self.assertTrue(300 <= best['axis_00'] <= 700) + self.assertTrue(0 <= best['axis_01'] <= 0.8) + self.assertTrue(3.5 <= best['axis_02'] <= 6.5) + + for status in df['status']: + self.assertTrue(status) + for loss in df['losses']: + self.assertTrue(isinstance(loss, float)) + + def test_getOptunitySolver(self): + config = { + "hyperparameter": { + "axis_00": { + "domain": "uniform", + "data": [300, 800], + "type": float + }, + "axis_01": { + "domain": "uniform", + "data": [-1, 1], + "type": float + }, + "axis_02": { + "domain": "uniform", + "data": [0, 10], + "type": float + } + }, + "max_iterations": 100 + } + + project = HyppopyProject(config) + solver = SolverPool.get("optunity", project) + self.assertTrue(isinstance(solver, OptunitySolver)) + vfunc = FunctionSimulator() + vfunc.load_default() + solver.blackbox = vfunc + solver.run(print_stats=False) + df, best = solver.get_results() + self.assertTrue(300 <= best['axis_00'] <= 800) + self.assertTrue(-1 <= best['axis_01'] <= 1) + self.assertTrue(0 <= best['axis_02'] <= 10) + + for status in df['status']: + self.assertTrue(status) + for loss in df['losses']: + self.assertTrue(isinstance(loss, float)) + + def test_getOptunaSolver(self): + config = { + "hyperparameter": { + "axis_00": { + "domain": "uniform", + "data": [300, 800], + "type": float + }, + "axis_01": { + "domain": "uniform", + "data": [-1, 1], + "type": float + }, + "axis_02": { + "domain": "uniform", + "data": [0, 10], + "type": float + } + }, + "max_iterations": 100 + } + + project = HyppopyProject(config) + solver = SolverPool.get("optuna", project) + self.assertTrue(isinstance(solver, OptunaSolver)) + vfunc = FunctionSimulator() + vfunc.load_default() + solver.blackbox = vfunc + solver.run(print_stats=False) + df, best = solver.get_results() + self.assertTrue(300 <= best['axis_00'] <= 800) + self.assertTrue(-1 <= best['axis_01'] <= 1) + self.assertTrue(0 <= best['axis_02'] <= 10) + + for status in df['status']: + self.assertTrue(status) + for loss in df['losses']: + self.assertTrue(isinstance(loss, float)) + + def test_getRandomsearchSolver(self): + config = { + "hyperparameter": { + "axis_00": { + "domain": "uniform", + "data": [0, 800], + "type": float + }, + "axis_01": { + "domain": "uniform", + "data": [-1, 1], + "type": float + }, + "axis_02": { + "domain": "uniform", + "data": [0, 10], + "type": float + } + }, + "max_iterations": 300 + } + + project = HyppopyProject(config) + solver = SolverPool.get("randomsearch", project) + self.assertTrue(isinstance(solver, RandomsearchSolver)) + vfunc = FunctionSimulator() + vfunc.load_default() + solver.blackbox = vfunc + solver.run(print_stats=False) + df, best = solver.get_results() + self.assertTrue(0 <= best['axis_00'] <= 800) + self.assertTrue(-1 <= best['axis_01'] <= 1) + self.assertTrue(0 <= best['axis_02'] <= 10) + + for status in df['status']: + self.assertTrue(status) + for loss in df['losses']: + self.assertTrue(isinstance(loss, float)) + + def test_getQuasiRandomsearchSolver(self): + config = { + "hyperparameter": { + "axis_00": { + "domain": "uniform", + "data": [0, 800], + "type": float + }, + "axis_01": { + "domain": "uniform", + "data": [-1, 1], + "type": float + }, + "axis_02": { + "domain": "uniform", + "data": [0, 10], + "type": float + } + }, + "max_iterations": 300 + } + + project = HyppopyProject(config) + solver = SolverPool.get("quasirandomsearch", project) + self.assertTrue(isinstance(solver, QuasiRandomsearchSolver)) + vfunc = FunctionSimulator() + vfunc.load_default() + solver.blackbox = vfunc + solver.run(print_stats=False) + df, best = solver.get_results() + self.assertTrue(0 <= best['axis_00'] <= 800) + self.assertTrue(-1 <= best['axis_01'] <= 1) + self.assertTrue(0 <= best['axis_02'] <= 10) + + for status in df['status']: + self.assertTrue(status) + for loss in df['losses']: + self.assertTrue(isinstance(loss, float)) + + def test_getQuasiRandomsearchSolver(self): + config = { + "hyperparameter": { + "value 1": { + "domain": "uniform", + "data": [0, 20], + "type": int, + "frequency": 11 + }, + "value 2": { + "domain": "normal", + "data": [0, 20.0], + "type": float, + "frequency": 11 + }, + "value 3": { + "domain": "loguniform", + "data": [1, 10000], + "type": float, + "frequency": 11 + }, + "categorical": { + "domain": "categorical", + "data": ["a", "b"], + "type": str, + "frequency": 1 + } + }} + res_labels = ['value 1', 'value 2', 'value 3', 'categorical'] + res_values = [[0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20], + [0.0, 5.467452952462635, 8.663855974622837, 9.755510546899107, 9.973002039367397, 10.0, + 10.026997960632603, 10.244489453100893, 11.336144025377163, 14.532547047537365, 20.0], + [1.0, 2.51188643150958, 6.309573444801933, 15.848931924611136, 39.810717055349734, + 100.00000000000004, 251.18864315095806, 630.9573444801938, 1584.8931924611143, + 3981.071705534977, 10000.00000000001], + ['a', 'b']] + project = HyppopyProject(config) + solver = SolverPool.get("gridsearch", project) + self.assertTrue(isinstance(solver, GridsearchSolver)) + searchspace = solver.convert_searchspace(config["hyperparameter"]) + for n in range(len(res_labels)): + self.assertEqual(res_labels[n], searchspace[0][n]) + for i in range(3): + self.assertAlmostEqual(res_values[i], searchspace[1][i]) + self.assertEqual(res_values[3], searchspace[1][3]) diff --git a/setup.py b/setup.py index a634d9a..8c2b997 100644 --- a/setup.py +++ b/setup.py @@ -1,54 +1,54 @@ import os from setuptools import setup, find_packages with open('README.md') as f: readme = f.read() with open('LICENSE') as f: license = f.read() -VERSION = "0.5.0.4" +VERSION = "0.5.0.5" ROOT = os.path.dirname(os.path.realpath(__file__)) new_init = [] with open(os.path.join(ROOT, *("hyppopy", "__init__.py")), "r") as infile: for line in infile: new_init.append(line) for n in range(len(new_init)): if new_init[n].startswith("__version__"): split = line.split("=") new_init[n] = "__version__ = '" + VERSION + "'\n" with open(os.path.join(ROOT, *("hyppopy", "__init__.py")), "w") as outfile: outfile.writelines(new_init) setup( name='hyppopy', version=VERSION, description='Hyper-Parameter Optimization Toolbox for Blackboxfunction Optimization', long_description=readme, # if you want, put your own name here # (this would likely result in people sending you emails) author='Sven Wanner', author_email='s.wanner@dkfz.de', url='', license=license, packages=find_packages(exclude=('tests', 'doc')), # the requirements to install this project. # Since this one is so simple this is empty. install_requires=[ 'bayesian-optimization>=1.0.1', 'hyperopt>=0.1.2', 'matplotlib>=3.0.3', 'numpy>=1.16.2', 'optuna>=0.9.0', 'Optunity>=1.1.1', 'pandas>=0.24.2', 'pytest>=4.3.1', 'scikit-learn>=0.20.3', 'scipy>=1.2.1', 'visdom>=0.1.8.8' ], )