diff --git a/hyppopy/BlackboxFunction.py b/hyppopy/BlackboxFunction.py index 729d984..ce5fde6 100644 --- a/hyppopy/BlackboxFunction.py +++ b/hyppopy/BlackboxFunction.py @@ -1,171 +1,183 @@ # Hyppopy - A Hyper-Parameter Optimization Toolbox # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE __all__ = ['BlackboxFunction'] import os import logging import functools from hyppopy.globals import DEBUGLEVEL LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) def default_kwargs(**defaultKwargs): """ Decorator defining default args in **kwargs arguments """ + def actual_decorator(fn): @functools.wraps(fn) def g(*args, **kwargs): defaultKwargs.update(kwargs) return fn(*args, **defaultKwargs) + return g + return actual_decorator class BlackboxFunction(object): """ This class is a BlackboxFunction wrapper class encapsulating the loss function. Additional function pointer can be set to get access at different pipelining steps: - dataloader_func: data loading, the function must return a data object and is called first when the solver is executed. The data object returned will be the input of the blackbox function. - preprocess_func: data preprocessing is called after dataloader_func, the functions signature must be foo(data, params) and must return a data object. The input is the data object set directly or via dataloader_func, the params are passed from constructor params. - callback_func: this function is called at each iteration step getting passed the trail info content, can be used for custom visualization - data: add a data object directly The constructor accepts several function pointers or a data object which are all None by default (see below). Additionally one can define an arbitrary number of arg pairs. These are passed as input to each function pointer as arguments. :param dataloader_func: data loading function pointer, default=None :param preprocess_func: data preprocessing function pointer, default=None :param callback_func: callback function pointer, default=None :param data: data object, default=None :param kwargs: additional arg=value pairs """ @default_kwargs(blackbox_func=None, dataloader_func=None, preprocess_func=None, callback_func=None, data=None) def __init__(self, **kwargs): self._blackbox_func = None self._preprocess_func = None self._dataloader_func = None self._callback_func = None self._raw_data = None self._data = None self.setup(kwargs) def __call__(self, **kwargs): """ Call method calls blackbox_func passing the data object and the args passed :param kwargs: [dict] args :return: blackbox_func(data, kwargs) """ - return self.blackbox_func(self.data, kwargs) + try: + try: + return self.blackbox_func(self.data, kwargs) + except: + return self.blackbox_func(self.data, **kwargs) + except: + try: + return self.blackbox_func(kwargs) + except: + return self.blackbox_func(**kwargs) def setup(self, kwargs): """ Alternative to Constructor, kwargs signature see __init__ :param kwargs: (see __init__) """ self._blackbox_func = kwargs['blackbox_func'] self._preprocess_func = kwargs['preprocess_func'] self._dataloader_func = kwargs['dataloader_func'] self._callback_func = kwargs['callback_func'] self._raw_data = kwargs['data'] self._data = self._raw_data del kwargs['blackbox_func'] del kwargs['preprocess_func'] del kwargs['dataloader_func'] del kwargs['data'] params = kwargs if self.dataloader_func is not None: self._raw_data = self.dataloader_func(params=params) - assert self._raw_data is not None, "Missing data exception!" - assert self.blackbox_func is not None, "Missing blackbox fucntion exception!" + # assert self._raw_data is not None, "Missing data exception!" + assert self.blackbox_func is not None, "Missing blackbox function exception!" if self.preprocess_func is not None: result = self.preprocess_func(data=self._raw_data, params=params) if result is not None: self._data = result else: self._data = self._raw_data else: self._data = self._raw_data @property def blackbox_func(self): """ BlackboxFunction wrapper class encapsulating the loss function or a function accepting a hyperparameter set and returning a float. :return: [object] pointer to blackbox_func """ return self._blackbox_func @property def preprocess_func(self): """ Data preprocessing is called after dataloader_func, the functions signature must be foo(data, params) and must return a data object. The input is the data object set directly or via dataloader_func, the params are passed from constructor params. :return: [object] preprocess_func """ return self._preprocess_func @property def dataloader_func(self): """ Data loading, the function must return a data object and is called first when the solver is executed. The data object returned will be the input of the blackbox function. :return: [object] dataloader_func """ return self._dataloader_func @property def callback_func(self): """ This function is called at each iteration step getting passed the trail info content, can be used for custom visualization :return: [object] callback_func """ return self._callback_func @property def raw_data(self): """ This data structure is used to store the return from dataloader_func to serve as input for preprocess_func if available. :return: [object] raw_data """ return self._raw_data @property def data(self): """ Datastructure keeping the input data. :return: [object] data """ return self._data diff --git a/hyppopy/CandidateDescriptor.py b/hyppopy/CandidateDescriptor.py new file mode 100644 index 0000000..5db69df --- /dev/null +++ b/hyppopy/CandidateDescriptor.py @@ -0,0 +1,108 @@ +class CandidateDescriptor(object): + """ + Descriptor that defines an candidate the solver wants to be checked. + It is used to lable/identify the candidates and their results in the case of batch processing. + """ + + def __init__(self, **definingValues): + """ + @param definingValues Class assumes that all variables passed to the computer are parameters of the candidate + the instance should represent. + """ + import uuid + + self._definingValues = definingValues + + self._definingStr = str() + + for item in sorted(definingValues.items()): + self._definingStr = self._definingStr + "'" + str(item[0]) + "':'" + str(item[1]) + "'," + + self.ID = str(uuid.uuid4()) + + def __missing__(self, key): + return None + + def __len__(self): + return len(self._definingValues) + + def __contains__(self, key): + return key in self._definingValues + + def __eq__(self, other): + if isinstance(other, self.__class__): + return self._definingValues == other._definingValues + else: + return False + + def __hash__(self): + return hash(self._definingStr) + + def __ne__(self, other): + return not self.__eq__(other) + + def __repr__(self): + return 'EvalInstanceDescriptor(%s)' % (self._definingValues) + + def __str__(self): + return '(%s)' % (self._definingValues) + + def keys(self): + return self._definingValues.keys() + + def __getitem__(self, key): + if key in self._definingValues: + return self._definingValues[key] + raise KeyError('Unkown defining value key was requested. Key: {}; self: {}'.format(key, self)) + + def get_values(self): + return self._definingValues + + +class CandicateDescriptorWrapper: + + class InternalCandidateValueWrapper: + def __init__(self, value_list): + self._value_list = value_list + + def __gt__(self, other): + boundary_condition = True + for value in self._value_list: + if value > other: + continue + else: + boundary_condition = False + break + return boundary_condition + + def __lt__(self, other): + boundary_condition = True + for value in self._value_list: + if value < other: + continue + else: + boundary_condition = False + break + return boundary_condition + + def get(self): + return self._value_list + + def __init__(self, keys): + self._cand = None + self._keys = keys + + def __iter__(self): + return iter(self._cand) + + def __getitem__(self, key): + return self.InternalCandidateValueWrapper([x[key] for x in self._cand]) + + def keys(self): + return self._keys + + def set(self, obj): + self._cand = obj + + def get(self): + return self._cand diff --git a/hyppopy/MPIBlackboxFunction.py b/hyppopy/MPIBlackboxFunction.py new file mode 100644 index 0000000..cb2c5b4 --- /dev/null +++ b/hyppopy/MPIBlackboxFunction.py @@ -0,0 +1,83 @@ +# Hyppopy - A Hyper-Parameter Optimization Toolbox +# +# Copyright (c) German Cancer Research Center, +# Division of Medical Image Computing. +# All rights reserved. +# +# This software is distributed WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. +# +# See LICENSE +from hyppopy.BlackboxFunction import BlackboxFunction + +__all__ = ['MPIBlackboxFunction'] + +import os +import logging +import functools +from hyppopy.globals import DEBUGLEVEL, MPI_TAGS +from mpi4py import MPI + +LOG = logging.getLogger(os.path.basename(__file__)) +LOG.setLevel(DEBUGLEVEL) + + +def default_kwargs(**defaultKwargs): + """ + Decorator defining default args in **kwargs arguments + """ + def actual_decorator(fn): + @functools.wraps(fn) + def g(*args, **kwargs): + defaultKwargs.update(kwargs) + return fn(*args, **defaultKwargs) + return g + return actual_decorator + + +class MPIBlackboxFunction(BlackboxFunction): + """ + This class is a BlackboxFunction wrapper class encapsulating the loss function. + # TODO: complete class documentation + The constructor accepts several function pointers or a data object which are all None by default (see below). + Additionally one can define an arbitrary number of arg pairs. These are passed as input to each function pointer as + arguments. + + :param dataloader_func: data loading function pointer, default=None + :param preprocess_func: data preprocessing function pointer, default=None + :param callback_func: callback function pointer, default=None + :param data: data object, default=None + :param mpi_comm: [MPI communicator] MPI communicator instance. If None, we create a new MPI.COMM_WORLD, default=None + :param kwargs: additional arg=value pairs + """ + + @default_kwargs(blackbox_func=None, dataloader_func=None, preprocess_func=None, callback_func=None, data=None, mpi_comm=None) + def __init__(self, **kwargs): + mpi_comm = kwargs['mpi_comm'] + del kwargs['mpi_comm'] + self._mpi_comm = None + + if mpi_comm is None: + print('MPIBlackboxFunction: No mpi_comm given: Using MPI.COMM_WORLD') + self._mpi_comm = MPI.COMM_WORLD + else: + self._mpi_comm = mpi_comm + + super().__init__(**kwargs) + + def call_batch(self, candidates): + results = dict() + size = self._mpi_comm.Get_size() + + for i, candidate in enumerate(candidates): + dest = (i % (size-1)) + 1 + self._mpi_comm.send(candidate, dest=dest, tag=MPI_TAGS.MPI_SEND_CANDIDATE.value) + + while True: + for i in range(size - 1): + if len(candidates) == len(results): + print('All results received!') + return results + cand_id, result_dict = MPI.COMM_WORLD.recv(source=i + 1, tag=MPI_TAGS.MPI_SEND_RESULTS.value) + results[cand_id] = result_dict \ No newline at end of file diff --git a/hyppopy/globals.py b/hyppopy/globals.py index 7d74106..1f966cf 100644 --- a/hyppopy/globals.py +++ b/hyppopy/globals.py @@ -1,34 +1,40 @@ # Hyppopy - A Hyper-Parameter Optimization Toolbox # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE import os import sys import logging +from enum import Enum ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) sys.path.insert(0, ROOT) LIBNAME = "hyppopy" TESTDATA_DIR = os.path.join(ROOT, *(LIBNAME, "tests", "data")) HYPERPARAMETERPATH = "hyperparameter" SETTINGSPATH = "settings" FUNCTIONSIMULATOR_DATAPATH = os.path.join(os.path.join(ROOT, LIBNAME), "virtualparameterspace") SUPPORTED_DOMAINS = ["uniform", "normal", "loguniform", "categorical"] SUPPORTED_DTYPES = ["int", "float", "str"] DEFAULTGRIDFREQUENCY = 10 LOGFILENAME = os.path.join(ROOT, '{}_log.log'.format(LIBNAME)) DEBUGLEVEL = logging.DEBUG logging.basicConfig(filename=LOGFILENAME, filemode='w', format='%(levelname)s: %(name)s - %(message)s') + + +class MPI_TAGS(Enum): + MPI_SEND_CANDIDATE = 55 + MPI_SEND_RESULTS = 99 diff --git a/hyppopy/solvers/DynamicPSOSolver.py b/hyppopy/solvers/DynamicPSOSolver.py new file mode 100644 index 0000000..ecf3d69 --- /dev/null +++ b/hyppopy/solvers/DynamicPSOSolver.py @@ -0,0 +1,250 @@ +# Hyppopy - A Hyper-Parameter Optimization Toolbox +# +# Copyright (c) German Cancer Research Center, +# Division of Medical Image Computing. +# All rights reserved. +# +# This software is distributed WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. +# +# See LICENSE + +import os +import sys +import numpy +import datetime +import logging +import optunity +from pprint import pformat + +from hyppopy.CandidateDescriptor import CandidateDescriptor, CandicateDescriptorWrapper +from hyppopy.globals import DEBUGLEVEL +from hyperopt import Trials + +LOG = logging.getLogger(os.path.basename(__file__)) +LOG.setLevel(DEBUGLEVEL) + +from hyppopy.solvers.HyppopySolver import HyppopySolver +from .OptunitySolver import OptunitySolver + +class DynamicPSOSolver(OptunitySolver): + """Dynamic PSO HyppoPy Solver Class""" + + def define_interface(self): + """ + Function called after instantiation to define individual parameters for child solver class by calling + _add_member function for each class member variable to be defined. When designing your own solver class, + you need to implement this method to define custom solver options that are automatically converted + to class attributes. + """ + super().define_interface() + self._add_method("update_param") # Pass function used to adapt parameters during dynamic PSO as specified by user. + self._add_method("combine_obj") # Pass function indicating how to combine obj. func. arguments and parameters to obtain scalar value. + self._add_member("num_args_obj", int) # Pass number of arguments/terms contributing to obj. func. + self._add_member("num_params_obj", int) # Pass number of parameters of obj. func. + self._add_member("phi1", float, default=1.5) # Pass first PSO acceleration coefficient. + self._add_member("phi2", float, default=2.0) # Pass second PSO acceleration coefficient. + self._add_hyperparameter_signature(name="domain", dtype=str, options=["uniform", "loguniform", "categorical"]) + + def _add_method(self, name, func=None, default=None): + """ + When designing your child solver class you need to implement the define_interface abstract method where you can + call _add_member_function to define custom solver options, here of Python callable type, which are automatically + converted to class methods. + + :param func: [callable] function object to be passed to solver + """ + assert isinstance(name, str), "Precondition violation, name needs to be of type str, got {}.".format(type(name)) + if func is not None: + assert callable(func), "Precondition violation, passed object is not callable!" + if default is not None: + assert callable(default), "Precondition violation, passed object is not callable!" + setattr(self, name, func) + self._child_members[name] = {"type": "callable", "function": func, "default": default} + + def convert_searchspace(self, hyperparameter): + """ + Get unified hyppopy-like parameter space description as input and, if necessary, + convert it into a solver-lib specific format. The function is invoked when run is called and what it returns + is passed as searchspace argument to the function execute_solver. + + :param hyperparameter: [dict] nested parameter description dict e.g. {'name': {'domain':'uniform', 'data':[0,1], 'type':float}, ...} + + :return: [object] converted hyperparameter space + :return: [dict] dict keeping domains for different hyperparameters. + """ + LOG.debug("convert input parameter\n\n\t{}\n".format(pformat(hyperparameter))) + # Split input in categorical and non-categorical data. + cat, uni = self.split_categorical(hyperparameter) + # Build up dict keeping all non-categorical data. + uniforms = {} + domains = {} + for key, value in uni.items(): + for key2, value2 in value.items(): + if key2 == "data": + if len(value2) == 3: + uniforms[key] = value2[0:2] + elif len(value2) == 2: + uniforms[key] = value2 + else: + raise AssertionError("precondition violation, optunity searchspace needs list with left and right range bounds!") + if key2 == "domain": + domains[key] = value2 + + if len(cat) == 0: + return uniforms, domains + # Build nested categorical structure. + inner_level = uniforms + for key, value in cat.items(): + tmp = {} + optunity_space = {} + for key2, value2 in value.items(): + if key2 == "data": + for elem in value2: + tmp[elem] = inner_level + if key2 == "domain": + domains[key] = value2 + optunity_space[key] = tmp + inner_level = optunity_space + return optunity_space, domains + + def hyppopy_optunity_solver_pmap(self, f, seq): + # Check if seq is empty. I so, return an empty result list. + if len(seq) == 0: + return [] + + candidates = [] + for elem in seq: + can = CandidateDescriptor(**elem) + candidates.append(can) + + cand_list = CandicateDescriptorWrapper(keys=seq[0].keys()) + cand_list.set(candidates) + + f_result = f(cand_list) + + # If one candidate does not match the constraints, f() returns a single default value. + # This is a problem as all the other candidates are not calculated either. + # The following is a workaround. We split the candidate_list into 2 lists and call the map function recursively until all valid parameters are processed. + if not isinstance(f_result, list): + # First half + seq_A = seq[:len(seq) // 2] + temp_result_a = self.hyppopy_optunity_solver_pmap(f, seq_A) + + seq_B = seq[len(seq) // 2:] + temp_result_b = self.hyppopy_optunity_solver_pmap(f, seq_B) + # f_result = [42] + + f_result = temp_result_a + temp_result_b + + return f_result + + def execute_solver(self, searchspace, domains): + """ + This function is called immediately after convert_searchspace and uses the output of the latter as input. Its + purpose is to call the solver lib's main optimization function. + + :param searchspace: converted hyperparameter space + """ + LOG.debug("execute_solver using solution space:\n\n\t{}\n".format(pformat(searchspace))) + tree = optunity.search_spaces.SearchTree(searchspace) # Set up tree structure to model search space. + box = tree.to_box() # Create set of box constraints to define given search space. + f = optunity.functions.logged(self.loss_function_batch) # Call log here because function signature used later on is internal logic. + f = tree.wrap_decoder(f) # Wrap decoder and constraints for internal search space rep. + f = optunity.constraints.wrap_constraints(f, default=sys.float_info.max*numpy.ones(self.num_args_obj), range_oo=box) + # 'wrap_constraints' decorates function f with given input domain constraints. default [float] gives a + # function value to default to in case of constraint violations. range_oo [dict] gives open range + # constraints lb and lu, i.e. lb < x < ub and range = (lb, ub), respectively. + + try: + self.best, _ = optunity.optimize_dyn_PSO(func=f, + box=box, + domains=domains, + maximize=False, + max_evals=self.max_iterations, + num_args_obj=self.num_args_obj, + num_params_obj=self.num_params_obj, + pmap=self.hyppopy_optunity_solver_pmap, #map,#optunity.pmap, + decoder=tree.decode, + update_param=self.update_param, + eval_obj=self.combine_obj, + phi1=self.phi1, + phi2=self.phi2 + ) + # Workaround: Unpack best result, im max_iterations was reached. + try: + for key in self.best: + self.best[key] = self.best[key].get()[0] + except: + pass + """ + optimize_dyn_PSO(func, maximize=False, max_evals=0, pmap=map, decoder=None, update_param=None, eval_obj=None) + Optimize func with dynamic PSO solver. + :param func: [callable] objective function + :param maximize: [bool] maximize or minimize + :param max_evals: [int] maximum number of permitted function evaluations + :param pmap: [function] map() function to use + :param update_param: [function] function to update parameters of objective function + based on current state of knowledge + :param eval_obj: [function] function giving functional form of objective function, i.e. + how to combine parameters and terms to obtain scalar fitness/loss. + + :return: solution, named tuple with further details + optimize_dyn_PSO function (api.py) internally uses 'optimize' function from dynamic PSO solver module. + """ + except Exception as e: + LOG.error("Internal error in optunity.optimize_dyn_PSO occured. {}".format(e)) + raise BrokenPipeError("Internal error in optunity.optimize_dyn_PSO occured. {}".format(e)) + + def print_best(self): + """ + Optimization result console output printing. + """ + print("\n") + print("#" * 40) + print("### Best Parameter Choice ###") + print("#" * 40) + for name, value in self.best.items(): + print(" - {}\t:\t{}".format(name, value)) + #print("\n - number of iterations\t:\t{}".format(self.trials.trials[-1]['tid']+1)) + #print(" - total time\t:\t{}d:{}h:{}m:{}s:{}ms".format(self._total_duration[0], + # self._total_duration[1], + # self._total_duration[2], + # self._total_duration[3], + # self._total_duration[4])) + print("#" * 40) + + def run(self, print_stats=True): + """ + This function starts the optimization process. + :param print_stats: [bool] en- or disable console output + """ + self._idx = 0 + self.trials = Trials() + + start_time = datetime.datetime.now() + try: + search_space, domains = self.convert_searchspace(self.project.hyperparameter) + except Exception as e: + msg = "Failed to convert searchspace, error: {}".format(e) + LOG.error(msg) + raise AssertionError(msg) + try: + self.execute_solver(search_space, domains) + except Exception as e: + msg = "Failed to execute solver, error: {}".format(e) + LOG.error(msg) + raise AssertionError(msg) + end_time = datetime.datetime.now() + dt = end_time - start_time + days = divmod(dt.total_seconds(), 86400) + hours = divmod(days[1], 3600) + minutes = divmod(hours[1], 60) + seconds = divmod(minutes[1], 1) + milliseconds = divmod(seconds[1], 0.001) + self._total_duration = [int(days[0]), int(hours[0]), int(minutes[0]), int(seconds[0]), int(milliseconds[0])] + self.print_best() + if print_stats: + self.print_timestats() + diff --git a/hyppopy/solvers/GridsearchSolver.py b/hyppopy/solvers/GridsearchSolver.py index f03a485..f11df27 100644 --- a/hyppopy/solvers/GridsearchSolver.py +++ b/hyppopy/solvers/GridsearchSolver.py @@ -1,234 +1,234 @@ # Hyppopy - A Hyper-Parameter Optimization Toolbox # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE import os import logging import warnings import numpy as np from pprint import pformat + from scipy.stats import norm from itertools import product from hyppopy.globals import DEBUGLEVEL, DEFAULTGRIDFREQUENCY from hyppopy.solvers.HyppopySolver import HyppopySolver +from hyppopy.CandidateDescriptor import CandidateDescriptor LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) def get_uniform_axis_sample(a, b, N, dtype): """ Returns a uniform sample x(n) in the range [a,b] sampled at N pojnts :param a: left value range bound :param b: right value range bound :param N: discretization of intervall [a,b] :param dtype: data type :return: [list] axis range """ assert a < b, "condition a < b violated!" assert isinstance(N, int), "condition N of type int violated!" if dtype is int: return list(np.linspace(a, b, N).astype(int)) elif dtype is float: return list(np.linspace(a, b, N)) else: raise AssertionError("dtype {} not supported for uniform sampling!".format(dtype)) def get_norm_cdf(N): """ Returns a normed gaussian cdf (range [0,1]) with N sampling points :param N: sampling points :return: [ndarray] gaussian cdf function values """ assert isinstance(N, int), "condition N of type int violated!" even = True if N % 2 != 0: N -= 1 even = False N = int(N/2) sigma = 1/3 x = np.linspace(0, 1, N) y1 = norm.cdf(x, loc=0, scale=sigma)-0.5 if not even: y1 = np.append(y1, [0.5]) y2 = 1-(norm.cdf(x, loc=0, scale=sigma)-0.5) y2 = np.flip(y2, axis=0) y = np.concatenate((y1, y2), axis=0) return y def get_gaussian_axis_sample(a, b, N, dtype): """ Returns a function value f(n) where f is a gaussian cdf in range [a, b] and N sampling points :param a: left value range bound :param b: right value range bound :param N: discretization of intervall [a,b] :param dtype: data type :return: [list] axis range """ assert a < b, "condition a < b violated!" assert isinstance(N, int), "condition N of type int violated!" data = [] for n in range(N): x = a + get_norm_cdf(N)[n]*(b-a) if dtype is int: data.append(int(x)) elif dtype is float: data.append(x) else: raise AssertionError("dtype {} not supported for uniform sampling!".format(dtype)) return data def get_logarithmic_axis_sample(a, b, N, dtype): """ Returns a function value f(n) where f is logarithmic function e^x sampling the exponent range [log(a), log(b)] linear at N sampling points. The function values returned are in the range [a, b]. :param a: left value range bound :param b: right value range bound :param N: discretization of intervall [a,b] :param dtype: data type :return: [list] axis range """ assert a < b, "condition a < b violated!" assert a > 0, "condition a > 0 violated!" assert isinstance(N, int), "condition N of type int violated!" # convert input range into exponent range lexp = np.log(a) rexp = np.log(b) exp_range = np.linspace(lexp, rexp, N) data = [] for n in range(exp_range.shape[0]): x = np.exp(exp_range[n]) if dtype is int: data.append(int(x)) elif dtype is float: data.append(x) else: raise AssertionError("dtype {} not supported for uniform sampling!".format(dtype)) return data class GridsearchSolver(HyppopySolver): """ The GridsearchSolver class implements a gridsearch optimization. The gridsearch supports categorical, uniform, normal and loguniform sampling. To use the GridsearchSolver, besides a range, one must specifiy the number of samples in the domain, e.g. 'data': [0, 1, 100] """ def __init__(self, project=None): """ The constructor accepts a HyppopyProject. :param project: [HyppopyProject] project instance, default=None """ HyppopySolver.__init__(self, project) def define_interface(self): """ This function is called when HyppopySolver.__init__ function finished. Child classes need to define their individual parameter here by calling the _add_member function for each class member variable need to be defined. Using _add_hyperparameter_signature the structure of a hyperparameter the solver expects must be defined. Both, members and hyperparameter signatures are later get checked, before executing the solver, ensuring settings passed fullfill solver needs. """ self._add_hyperparameter_signature(name="domain", dtype=str, options=["uniform", "normal", "loguniform", "categorical"]) self._add_hyperparameter_signature(name="data", dtype=list) self._add_hyperparameter_signature(name="frequency", dtype=int) self._add_hyperparameter_signature(name="type", dtype=type) - def loss_function_call(self, params): + def get_candidates(self, searchspace): """ - This function is called within the function loss_function and encapsulates the actual blackbox function call - in each iteration. The function loss_function takes care of the iteration driving and reporting, but each solver - lib might need some special treatment between the parameter set selection and the calling of the actual blackbox - function, e.g. parameter converting. - - :param params: [dict] hyperparameter space sample e.g. {'p1': 0.123, 'p2': 3.87, ...} + This function converts the searchspace to a candidate_list that can then be used to distribute via MPI. - :return: [float] loss + :param searchspace: converted hyperparameter space """ - loss = self.blackbox(**params) - if loss is None: - return np.nan - return loss + candidates_list = list() + candidates = [x for x in product(*searchspace[1])] + for c in candidates: + params = {} + for name, value in zip(searchspace[0], c): + params[name] = value + candidates_list.append(CandidateDescriptor(**params)) + + return candidates_list def execute_solver(self, searchspace): """ This function is called immediately after convert_searchspace and get the output of the latter as input. It's purpose is to call the solver libs main optimization function. :param searchspace: converted hyperparameter space """ - for x in product(*searchspace[1]): - params = {} - for name, value in zip(searchspace[0], x): - params[name] = value - try: - self.loss_function(**params) - except Exception as e: - msg = "internal error in randomsearch execute_solver occured. {}".format(e) - LOG.error(msg) - raise BrokenPipeError(msg) + candidates = self.get_candidates(searchspace) + + try: + self.loss_function_batch(candidates) + except Exception as e: + msg = "internal error in gridsearch execute_solver occured. {}".format(e) + LOG.error(msg) + raise BrokenPipeError(msg) self.best = self._trials.argmin def convert_searchspace(self, hyperparameter): """ The function converts the standard parameter input into a range list depending on the domain. These rangelists are later used with itertools product to create a paramater space sample of each combination. :param hyperparameter: [dict] hyperparameter space :return: [list] name and range for each parameter space axis """ LOG.debug("convert input parameter\n\n\t{}\n".format(pformat(hyperparameter))) searchspace = [[], []] for name, param in hyperparameter.items(): if param["domain"] != "categorical" and "frequency" not in param.keys(): param["frequency"] = DEFAULTGRIDFREQUENCY warnings.warn("No frequency field found, used default gridsearch frequency {}".format(DEFAULTGRIDFREQUENCY)) if param["domain"] == "categorical": searchspace[0].append(name) searchspace[1].append(param["data"]) elif param["domain"] == "uniform": searchspace[0].append(name) searchspace[1].append(get_uniform_axis_sample(param["data"][0], param["data"][1], param["frequency"], param["type"])) elif param["domain"] == "normal": searchspace[0].append(name) searchspace[1].append(get_gaussian_axis_sample(param["data"][0], param["data"][1], param["frequency"], param["type"])) elif param["domain"] == "loguniform": searchspace[0].append(name) searchspace[1].append(get_logarithmic_axis_sample(param["data"][0], param["data"][1], param["frequency"], param["type"])) return searchspace diff --git a/hyppopy/solvers/HyperoptSolver.py b/hyppopy/solvers/HyperoptSolver.py index 7540c71..bc8fdc8 100644 --- a/hyppopy/solvers/HyperoptSolver.py +++ b/hyppopy/solvers/HyperoptSolver.py @@ -1,202 +1,236 @@ # Hyppopy - A Hyper-Parameter Optimization Toolbox # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE import os import copy import logging import numpy as np from pprint import pformat from hyperopt import fmin, tpe, hp, STATUS_OK, STATUS_FAIL, Trials from hyppopy.globals import DEBUGLEVEL from hyppopy.solvers.HyppopySolver import HyppopySolver from hyppopy.BlackboxFunction import BlackboxFunction LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) class HyperoptSolver(HyppopySolver): def __init__(self, project=None): """ The constructor accepts a HyppopyProject. :param project: [HyppopyProject] project instance, default=None """ HyppopySolver.__init__(self, project) self._searchspace = None def define_interface(self): """ This function is called when HyppopySolver.__init__ function finished. Child classes need to define their individual parameter here by calling the _add_member function for each class member variable need to be defined. Using _add_hyperparameter_signature the structure of a hyperparameter the solver expects must be defined. Both, members and hyperparameter signatures are later get checked, before executing the solver, ensuring settings passed fullfill solver needs. """ self._add_member("max_iterations", int) self._add_hyperparameter_signature(name="domain", dtype=str, options=["uniform", "normal", "loguniform", "categorical"]) self._add_hyperparameter_signature(name="data", dtype=list) self._add_hyperparameter_signature(name="type", dtype=type) def loss_function(self, params): """ Loss function wrapper function. :param params: [dict] hyperparameter set :return: [float] loss """ for name, p in self._searchspace.items(): if p["domain"] != "categorical": if params[name] < p["data"][0]: params[name] = p["data"][0] if params[name] > p["data"][1]: params[name] = p["data"][1] status = STATUS_FAIL try: loss = self.blackbox(**params) if loss is not None: status = STATUS_OK else: loss = 1e9 except Exception as e: LOG.error("execution of self.blackbox(**params) failed due to:\n {}".format(e)) status = STATUS_FAIL loss = 1e9 cbd = copy.deepcopy(params) cbd['iterations'] = self._trials.trials[-1]['tid'] + 1 cbd['loss'] = loss cbd['status'] = status cbd['book_time'] = self._trials.trials[-1]['book_time'] cbd['refresh_time'] = self._trials.trials[-1]['refresh_time'] if isinstance(self.blackbox, BlackboxFunction) and self.blackbox.callback_func is not None: self.blackbox.callback_func(**cbd) if self._visdom_viewer is not None: self._visdom_viewer.update(cbd) return {'loss': loss, 'status': status} + def loss_func_cand_preprocess(self, params): + """ + Loss function wrapper function. + + :param params: [dict] hyperparameter set + + :return: [float] loss + """ + for name, p in self._searchspace.items(): + if p["domain"] != "categorical": + if params[name] < p["data"][0]: + params[name] = p["data"][0] + if params[name] > p["data"][1]: + params[name] = p["data"][1] + + return params + + def loss_func_postprocess(self, loss): + """ + Loss function wrapper function. + + :param params: [dict] hyperparameter set + + :return: [float] loss + """ + + if loss is not None: + status = STATUS_OK + else: + loss = 1e9 + + # return {'loss': loss, 'status': status} + return loss + def execute_solver(self, searchspace): """ This function is called immediately after convert_searchspace and get the output of the latter as input. It's purpose is to call the solver libs main optimization function. :param searchspace: converted hyperparameter space """ LOG.debug("execute_solver using solution space:\n\n\t{}\n".format(pformat(searchspace))) self.trials = Trials() try: self.best = fmin(fn=self.loss_function, space=searchspace, algo=tpe.suggest, max_evals=self.max_iterations, trials=self.trials) except Exception as e: msg = "internal error in hyperopt.fmin occured. {}".format(e) LOG.error(msg) raise BrokenPipeError(msg) def convert_searchspace(self, hyperparameter): """ This function gets the unified hyppopy-like parameterspace description as input and, if necessary, should convert it into a solver lib specific format. The function is invoked when run is called and what it returns is passed as searchspace argument to the function execute_solver. :param hyperparameter: [dict] nested parameter description dict e.g. {'name': {'domain':'uniform', 'data':[0,1], 'type':'float'}, ...} :return: [object] converted hyperparameter space """ self._searchspace = hyperparameter solution_space = {} for name, content in hyperparameter.items(): param_settings = {'name': name} for key, value in content.items(): if key == 'domain': param_settings['domain'] = value elif key == 'data': param_settings['data'] = value elif key == 'type': param_settings['dtype'] = value solution_space[name] = self.convert(param_settings) return solution_space def convert(self, param_settings): """ Convert searchspace to hyperopt specific searchspace :param param_settings: [dict] hyperparameter description :return: [object] hyperopt description """ name = param_settings["name"] domain = param_settings["domain"] dtype = param_settings["dtype"] data = param_settings["data"] if domain == "uniform": if dtype is float: return hp.uniform(name, data[0], data[1]) elif dtype is int: data = list(np.arange(int(data[0]), int(data[1] + 1))) return hp.choice(name, data) else: msg = "cannot convert the type {} in domain {}".format(dtype, domain) LOG.error(msg) raise LookupError(msg) elif domain == "loguniform": if dtype is float: if data[0] == 0: data[0] += 1e-23 assert data[0] > 0, "precondition Violation, a < 0!" assert data[0] < data[1], "precondition Violation, a > b!" assert data[1] > 0, "precondition Violation, b < 0!" lexp = np.log(data[0]) rexp = np.log(data[1]) assert lexp is not np.nan, "precondition violation, left bound input error, results in nan!" assert rexp is not np.nan, "precondition violation, right bound input error, results in nan!" return hp.loguniform(name, lexp, rexp) else: msg = "cannot convert the type {} in domain {}".format(dtype, domain) LOG.error(msg) raise LookupError(msg) elif domain == "normal": if dtype is float: mu = (data[1] - data[0]) / 2.0 sigma = mu / 3 return hp.normal(name, data[0] + mu, sigma) else: msg = "cannot convert the type {} in domain {}".format(dtype, domain) LOG.error(msg) raise LookupError(msg) elif domain == "categorical": if dtype is str: return hp.choice(name, data) elif dtype is bool: conv = [] for elem in data: if elem == "true" or elem == "True" or elem == 1 or elem == "1" or elem == True: conv.append(True) elif elem == "false" or elem == "False" or elem == 0 or elem == "0" or elem == False: conv.append(False) else: msg = "cannot convert the type {} in domain {}, unknown bool type value".format(dtype, domain) LOG.error(msg) raise LookupError(msg) return hp.choice(name, conv) else: msg = "Precondition violation, domain named {} not available!".format(domain) LOG.error(msg) raise IOError(msg) diff --git a/hyppopy/solvers/HyppopySolver.py b/hyppopy/solvers/HyppopySolver.py index 2ffbd7d..aa80d61 100644 --- a/hyppopy/solvers/HyppopySolver.py +++ b/hyppopy/solvers/HyppopySolver.py @@ -1,512 +1,612 @@ # Hyppopy - A Hyper-Parameter Optimization Toolbox # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE +from _pytest import deprecated + +from hyppopy import CandidateDescriptor __all__ = ['HyppopySolver'] import abc import copy import types import datetime import numpy as np import pandas as pd from hyperopt import Trials from hyppopy.globals import * +from hyppopy.CandidateDescriptor import CandidateDescriptor from hyppopy.VisdomViewer import VisdomViewer from hyppopy.HyppopyProject import HyppopyProject from hyppopy.BlackboxFunction import BlackboxFunction +from hyppopy.MPIBlackboxFunction import MPIBlackboxFunction from hyppopy.FunctionSimulator import FunctionSimulator from hyppopy.globals import DEBUGLEVEL LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) class HyppopySolver(object): """ The HyppopySolver class is the base class for all solver addons. It defines virtual functions a child class has to implement to deal with the front-end communication, orchestrating the optimization process and ensuring a proper process information storing. The key idea is that the HyppopySolver class defines an interface to configure and run an object instance of itself independently from the concrete solver lib used to optimize in the background. To achieve this goal an addon developer needs to implement the abstract methods 'convert_searchspace', 'execute_solver' and 'loss_function_call'. These methods abstract the peculiarities of the solver libs to offer, on the user side, a simple and consistent parameter space configuration and optimization procedure. The method 'convert_searchspace' transforms the hyppopy parameter space description into the solver lib specific description. The method loss_function_call is used to handle solver lib specifics of calling the actual blackbox function and execute_solver is executed when the run method is invoked und takes care of calling the solver lib solving routine. The class HyppopySolver defines an interface to be implemented when writing a custom solver. Each solver derivative needs to implement the abstract methods: - convert_searchspace - execute_solver - loss_function_call + - TODO - define_interface The dev-user interface consists of the methods: - _add_member - _add_hyperparameter_signature - _check_project The end-user interface consists of the methods: - run - get_results - print_best - print_timestats - start_viewer """ def __init__(self, project=None): """ The constructor accepts a HyppopyProject. :param project: [HyppopyProject] project instance, default=None """ - self._idx = None # current iteration counter + self._idx = 0 # current iteration counter self._best = None # best parameter set self._trials = None # trials object, hyppopy uses the Trials object from hyperopt self._blackbox = None # blackbox function, eiter a function or a BlackboxFunction instance self._total_duration = None # keeps track of the solvers running time self._solver_overhead = None # stores the time overhead of the solver, means total time minus time in blackbox self._time_per_iteration = None # mean time per iterration self._accumulated_blackbox_time = None # summed time the solver was in the blackbox function self._visdom_viewer = None # visdom viewer instance self._child_members = {} # this dict keeps track of the settings the child solver defines self._hopt_signatures = {} # this dict keeps track of the hyperparameter signatures the child solver defines self.define_interface() # the child define interface function is called which defines settings and hyperparameter signatures if project is not None: self.project = project @abc.abstractmethod def convert_searchspace(self, hyperparameter): """ This function gets the unified hyppopy-like parameterspace description as input and, if necessary, should convert it into a solver lib specific format. The function is invoked when run is called and what it returns is passed as searchspace argument to the function execute_solver. :param hyperparameter: [dict] nested parameter description dict e.g. {'name': {'domain':'uniform', 'data':[0,1], 'type':'float'}, ...} :return: [object] converted hyperparameter space """ raise NotImplementedError('users must define convert_searchspace to use this class') @abc.abstractmethod def execute_solver(self, searchspace): """ This function is called immediately after convert_searchspace and get the output of the latter as input. It's purpose is to call the solver libs main optimization function. :param searchspace: converted hyperparameter space """ raise NotImplementedError('users must define execute_solver to use this class') @abc.abstractmethod - def loss_function_call(self, params): + def loss_function_call(self, params): # TODO: Delete me... """ This function is called within the function loss_function and encapsulates the actual blackbox function call in each iteration. The function loss_function takes care of the iteration driving and reporting, but each solver lib might need some special treatment between the parameter set selection and the calling of the actual blackbox function, e.g. parameter converting. :param params: [dict] hyperparameter space sample e.g. {'p1': 0.123, 'p2': 3.87, ...} :return: [float] loss """ + + # TODO This is deprecated! Mark or remove... raise NotImplementedError('users must define loss_function_call to use this class') + @abc.abstractmethod + def loss_function_batch_call(self, candidates): # TODO: Delete me... + """ + TODO + :param candidates: + :return: + """ + + # TODO This is deprecated! Mark or remove... + raise NotImplementedError('users must define loss_function_batch_call to use this class') + + def loss_func_cand_preprocess(self, candidates): # TODO: Delete me... + """ + TODO + :param candidates: + :return: + """ + # User may implement this function to preprocess candidates before calling the actual loss_function + # raise NotImplementedError('users must define loss_function_batch_call to use this class') + return candidates + + def loss_func_postprocess(self, results): # TODO: Delete me... + """ + TODO + :param candidates: + :return: + """ + # User may implement this function to postprocess results after calling the actual loss_function + # raise NotImplementedError('users must define loss_function_batch_call to use this class') + return results + + @abc.abstractmethod def define_interface(self): """ This function is called when HyppopySolver.__init__ function finished. Child classes need to define their individual parameter here by calling the _add_member function for each class member variable need to be defined. Using _add_hyperparameter_signature the structure of a hyperparameter the solver expects must be defined. Both, members and hyperparameter signatures are later get checked, before executing the solver, ensuring settings passed fullfill solver needs. """ raise NotImplementedError('users must define define_interface to use this class') def _add_member(self, name, dtype, value=None, default=None): """ When designing your child solver class you need to implement the define_interface abstract method where you can call _add_member to define custom solver options that are automatically converted to class attributes. :param name: [str] option name :param dtype: [type] option data type :param value: [object] option value :param default: [object] option default value """ assert isinstance(name, str), "precondition violation, name needs to be of type str, got {}".format(type(name)) if value is not None: assert isinstance(value, dtype), "precondition violation, value does not match dtype condition!" if default is not None: assert isinstance(default, dtype), "precondition violation, default does not match dtype condition!" setattr(self, name, value) self._child_members[name] = {"type": dtype, "value": value, "default": default} def _add_hyperparameter_signature(self, name, dtype, options=None): """ When designing your child solver class you need to implement the define_interface abstract method where you can call _add_hyperparameter_signature to define a hyperparamter signature which is automatically checked for consistency while solver execution. :param name: [str] hyperparameter name :param dtype: [type] hyperparameter data type :param options: [list] list of possible values the hp can be set, if None no option check is done """ assert isinstance(name, str), "precondition violation, name needs to be of type str, got {}".format(type(name)) self._hopt_signatures[name] = {"type": dtype, "options": options} def _check_project(self): """ The function checks the members and hyperparameter signatures read from the project instance to be consistent with the members and signatures defined in the child class via define_interface. """ assert isinstance(self.project, HyppopyProject), "Invalid project instance, either not set or setting failed!" # check hyperparameter signatures for name, param in self.project.hyperparameter.items(): for sig, settings in self._hopt_signatures.items(): if sig not in param.keys(): msg = "Missing hyperparameter signature {}!".format(sig) LOG.error(msg) raise LookupError(msg) else: if not isinstance(param[sig], settings["type"]): msg = "Hyperparameter signature type mismatch, expected type {} got {}!".format(settings["type"], param[sig]) LOG.error(msg) raise TypeError(msg) if settings["options"] is not None: if param[sig] not in settings["options"]: msg = "Wrong signature value, {} not found in signature options!".format(param[sig]) LOG.error(msg) raise LookupError(msg) # check child members for name in self._child_members.keys(): if name not in self.project.__dict__.keys(): msg = "missing settings field {}!".format(name) LOG.error(msg) raise LookupError(msg) self.__dict__[name] = self.project.settings[name] def __compute_time_statistics(self): """ Evaluates all timestatistic values available """ dts = [] for trial in self._trials.trials: if 'book_time' in trial.keys() and 'refresh_time' in trial.keys(): dt = trial['refresh_time'] - trial['book_time'] dts.append(dt.total_seconds()) self._time_per_iteration = np.mean(dts) * 1e3 self._accumulated_blackbox_time = np.sum(dts) * 1e3 tmp = self.total_duration - self._accumulated_blackbox_time self._solver_overhead = int(np.round(100.0 / (self.total_duration + 1e-12) * tmp)) def loss_function(self, **params): """ This function is called each iteration with a selected parameter set. The parameter set selection is driven by - the solver lib itself. The purpose of this function is to take care of the iteration reporting and the calling - of the callback_func is available. As a developer you might want to overwrite this function completely (e.g. - HyperoptSolver) but then you need to take care for iteration reporting for yourself. The alternative is to only - implement loss_function_call (e.g. OptunitySolver). + the solver lib itself. + This function just calls loss_function_batch() with a batch size of one. It takes care of converting the params to CandidateDescriptors. :param params: [dict] hyperparameter space sample e.g. {'p1': 0.123, 'p2': 3.87, ...} :return: [float] loss """ - self._idx += 1 - vals = {} - idx = {} - for key, value in params.items(): - vals[key] = [value] - idx[key] = [self._idx] - trial = {'tid': self._idx, - 'result': {'loss': None, 'status': 'ok'}, - 'misc': { - 'tid': self._idx, - 'idxs': idx, - 'vals': vals - }, - 'book_time': datetime.datetime.now(), - 'refresh_time': None - } + + newCandidate = CandidateDescriptor(**params) + results = self.loss_function_batch([newCandidate]) + + return list(results.values())[0]['loss'] # Here 'results' will always contain a single dict. We extract the loss from it and return it. + + def loss_function_batch(self, candidates): + """ + This function is called with a list of candidates. This list is driven by the solver lib itself. + The purpose of this function is to take care of the iteration reporting and the calling + of the callback_func if available. As a developer you might want to overwrite this function (or the 'non-batch'-version completely (e.g. + HyperoptSolver) but then you need to take care for iteration reporting for yourself. The alternative is to only + implement loss_function_call (e.g. OptunitySolver). + + :param candidates: [list of CandidateDescriptors] + + :return: [dict] result e.g. {'loss': 0.5, 'book_time': ..., 'refresh_time': ...} + """ + + # print('hello'*10) + results = dict() try: - loss = self.loss_function_call(params) - trial['result']['loss'] = loss - trial['result']['status'] = 'ok' - if loss is np.nan: - trial['result']['status'] = 'failed' + candidates = self.loss_func_cand_preprocess(candidates) + results = self.blackbox.call_batch(candidates) + if results is None: + results = np.nan + results = self.loss_func_postprocess(results) + except ZeroDivisionError as e: + # Fallback: If call_batch is not supported in BlackboxFunction, we iterate over the candidates in the batch. + message = "Script not started via MPI:\n {}".format(e) + LOG.error(message) + print(message) except Exception as e: - LOG.error("computing loss failed due to:\n {}".format(e)) - loss = np.nan - trial['result']['loss'] = np.nan - trial['result']['status'] = 'failed' - trial['refresh_time'] = datetime.datetime.now() - self._trials.trials.append(trial) - cbd = copy.deepcopy(params) - cbd['iterations'] = self._idx - cbd['loss'] = loss - cbd['status'] = trial['result']['status'] - cbd['book_time'] = trial['book_time'] - cbd['refresh_time'] = trial['refresh_time'] - if isinstance(self.blackbox, BlackboxFunction) and self.blackbox.callback_func is not None: - self.blackbox.callback_func(**cbd) - if self._visdom_viewer is not None: - self._visdom_viewer.update(cbd) - return loss + message = "call_batch not supported in BlackboxFunction:\n {}".format(e) + LOG.error(message) + print(message) + finally: + for i, candidate in enumerate(candidates): + cand_id = candidate.ID + # params = candidate.get_values() + + cand_results = dict() + cand_results['book_time'] = datetime.datetime.now() + try: + preprocessed_candidate_list = self.loss_func_cand_preprocess([candidate]) + candidate = preprocessed_candidate_list[0] + params = candidate.get_values() + try: + loss = self.blackbox(**params) + except: + loss = self.blackbox(params) + if loss is None: + loss = np.nan + cand_results['loss'] = loss + except Exception as e: + LOG.error("computing loss failed due to:\n {}".format(e)) + cand_results['loss'] = np.nan + cand_results['refresh_time'] = datetime.datetime.now() + results[cand_id] = cand_results + results = self.loss_func_postprocess(results) + + # initialize trials + for i, candidate in enumerate(candidates): + self._idx += 1 + vals = {} + idx = {} + for key in candidate.keys(): + vals[key] = [candidate[key]] + idx[key] = [self._idx] + trial = {'tid': self._idx, + 'result': {'loss': None, 'status': 'ok'}, + 'misc': { + 'tid': self._idx, + 'idxs': idx, + 'vals': vals + }, + 'book_time': results[candidate.ID]['book_time'], + 'refresh_time': results[candidate.ID]['refresh_time'] + } + try: + loss = results[candidate.ID]['loss'] + trial['result']['loss'] = loss + trial['result']['status'] = 'ok' + if loss is np.nan: + trial['result']['status'] = 'failed' + except Exception as e: + LOG.error("computing loss failed due to:\n {}".format(e)) + loss = np.nan + trial['result']['loss'] = np.nan + trial['result']['status'] = 'failed' + self._trials.trials.append(trial) + cbd = copy.deepcopy(candidate.get_values()) + cbd['iterations'] = self._idx + cbd['loss'] = loss + cbd['status'] = trial['result']['status'] + cbd['book_time'] = trial['book_time'] + cbd['refresh_time'] = trial['refresh_time'] + if (isinstance(self.blackbox, BlackboxFunction) or isinstance(self.blackbox, MPIBlackboxFunction)) and self.blackbox.callback_func is not None: + self.blackbox.callback_func(**cbd) + + return results def run(self, print_stats=True): """ This function starts the optimization process. :param print_stats: [bool] en- or disable console output """ self._idx = 0 self.trials = Trials() start_time = datetime.datetime.now() try: search_space = self.convert_searchspace(self.project.hyperparameter) except Exception as e: msg = "Failed to convert searchspace, error: {}".format(e) LOG.error(msg) raise AssertionError(msg) try: self.execute_solver(search_space) except Exception as e: msg = "Failed to execute solver, error: {}".format(e) LOG.error(msg) raise AssertionError(msg) end_time = datetime.datetime.now() dt = end_time - start_time days = divmod(dt.total_seconds(), 86400) hours = divmod(days[1], 3600) minutes = divmod(hours[1], 60) seconds = divmod(minutes[1], 1) milliseconds = divmod(seconds[1], 0.001) self._total_duration = [int(days[0]), int(hours[0]), int(minutes[0]), int(seconds[0]), int(milliseconds[0])] if print_stats: self.print_best() self.print_timestats() def get_results(self): """ This function returns a complete optimization history as pandas DataFrame and a dict with the optimal parameter set. :return: [DataFrame], [dict] history and optimal parameter set """ assert isinstance(self.trials, Trials), "precondition violation, wrong trials type! Maybe solver was not yet executed?" results = {'duration': [], 'losses': [], 'status': []} pset = self.trials.trials[0]['misc']['vals'] for p in pset.keys(): results[p] = [] for n, trial in enumerate(self.trials.trials): t1 = trial['book_time'] t2 = trial['refresh_time'] results['duration'].append((t2 - t1).microseconds / 1000.0) results['losses'].append(trial['result']['loss']) results['status'].append(trial['result']['status'] == 'ok') losses = np.array(results['losses']) results['losses'] = list(losses) pset = trial['misc']['vals'] for p in pset.items(): results[p[0]].append(p[1][0]) return pd.DataFrame.from_dict(results), self.best def print_best(self): """ Optimization result console output printing. """ print("\n") print("#" * 40) print("### Best Parameter Choice ###") print("#" * 40) for name, value in self.best.items(): print(" - {}\t:\t{}".format(name, value)) + print("\n - number of iterations\t:\t{}".format(self.trials.trials[-1]['tid']+1)) print(" - total time\t:\t{}d:{}h:{}m:{}s:{}ms".format(self._total_duration[0], self._total_duration[1], self._total_duration[2], self._total_duration[3], self._total_duration[4])) print("#" * 40) def print_timestats(self): """ Time statistic console output printing. """ print("\n") print("#" * 40) print("### Timing Statistics ###") print("#" * 40) print(" - per iteration: {}ms".format(int(self.time_per_iteration*1e4)/10000)) print(" - total time: {}d:{}h:{}m:{}s:{}ms".format(self._total_duration[0], self._total_duration[1], self._total_duration[2], self._total_duration[3], self._total_duration[4])) print("#" * 40) print(" - solver overhead: {}%".format(self.solver_overhead)) def start_viewer(self, port=8097, server="http://localhost"): """ Starts the visdom viewer. :param port: [int] port number, default: 8097 :param server: [str] server name, default: http://localhost """ try: self._visdom_viewer = VisdomViewer(self._project, port, server) except Exception as e: import warnings warnings.warn("Failed starting VisdomViewer. Is the server running? If not start it via $visdom") LOG.error("Failed starting VisdomViewer: {}".format(e)) self._visdom_viewer = None @property def project(self): """ HyppopyProject instance :return: [HyppopyProject] project instance """ return self._project @project.setter def project(self, value): """ Set HyppopyProject instance :param value: [HyppopyProject] project instance """ if isinstance(value, dict): self._project = HyppopyProject(value) elif isinstance(value, HyppopyProject): self._project = value else: msg = "Input error, project_manager of type: {} not allowed!".format(type(value)) LOG.error(msg) raise TypeError(msg) self._check_project() @property def blackbox(self): """ Get the BlackboxFunction object. :return: [object] BlackboxFunction instance or function """ return self._blackbox @blackbox.setter def blackbox(self, value): """ Set the BlackboxFunction wrapper class encapsulating the loss function or a function accepting a hyperparameter set and returning a float. :return: [object] pointer to blackbox_func """ - if isinstance(value, types.FunctionType) or isinstance(value, BlackboxFunction) or isinstance(value, FunctionSimulator): + if isinstance(value, types.FunctionType) or isinstance(value, BlackboxFunction) or isinstance(value, FunctionSimulator) or isinstance(value, MPIBlackboxFunction): self._blackbox = value else: self._blackbox = None msg = "Input error, blackbox of type: {} not allowed!".format(type(value)) LOG.error(msg) raise TypeError(msg) @property def best(self): """ Returns best parameter set. :return: [dict] best parameter set """ return self._best @best.setter def best(self, value): """ Set the best parameter set. :param value: [dict] best parameter set """ if not isinstance(value, dict): msg = "Input error, best of type: {} not allowed!".format(type(value)) LOG.error(msg) raise TypeError(msg) self._best = value @property def trials(self): """ Get the Trials instance. :return: [object] Trials instance """ return self._trials @trials.setter def trials(self, value): """ Set the Trials object. :param value: [object] Trials instance """ self._trials = value @property def total_duration(self): """ Get total computation duration. :return: [float] total computation time """ return (self._total_duration[0]*86400 + self._total_duration[1] * 3600 + self._total_duration[2] * 60 + self._total_duration[3]) * 1000 + self._total_duration[4] @property def solver_overhead(self): """ Get the solver overhead, this is the total time minus the duration of the blackbox function calls. :return: [float] solver overhead duration """ if self._solver_overhead is None: self.__compute_time_statistics() return self._solver_overhead @property def time_per_iteration(self): """ Get the mean duration per iteration. :return: [float] time per iteration """ if self._time_per_iteration is None: self.__compute_time_statistics() return self._time_per_iteration @property def accumulated_blackbox_time(self): """ Get the summed blackbox function computation time. :return: [float] blackbox function computation time """ if self._accumulated_blackbox_time is None: self.__compute_time_statistics() return self._accumulated_blackbox_time diff --git a/hyppopy/solvers/MPISolverWrapper.py b/hyppopy/solvers/MPISolverWrapper.py new file mode 100644 index 0000000..e95b841 --- /dev/null +++ b/hyppopy/solvers/MPISolverWrapper.py @@ -0,0 +1,166 @@ +# Hyppopy - A Hyper-Parameter Optimization Toolbox +# +# Copyright (c) German Cancer Research Center, +# Division of Medical Image Computing. +# All rights reserved. +# +# This software is distributed WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. +# +# See LICENSE +import datetime +import os +import logging + +import numpy as np +from mpi4py import MPI +from hyppopy.globals import DEBUGLEVEL, MPI_TAGS + +LOG = logging.getLogger(os.path.basename(__file__)) +LOG.setLevel(DEBUGLEVEL) + + +class MPISolverWrapper: + """ + TODO Class description + The MPISolverWrapper class wraps the functionality of solvers in Hyppopy to extend them with MPI functionality. + It builds upon the interface defined by the HyppopySolver class. + """ + def __init__(self, solver=None, mpi_comm=None): + """ + The constructor accepts a HyppopySolver. + + :param solver: [HyppopySolver] solver instance, default=None + :param mpi_comm: [MPI communicator] MPI communicator instance. If None, we create a new MPI.COMM_WORLD, default=None + """ + self._solver = solver + self._mpi_comm = None + if mpi_comm is None: + print('MPISolverWrapper: No mpi_comm given: Using MPI.COMM_WORLD') + self._mpi_comm = MPI.COMM_WORLD + else: + self._mpi_comm = mpi_comm + + @property + def blackbox(self): + """ + Get the BlackboxFunction object. + + :return: [object] BlackboxFunction instance or function of member solver + """ + return self._solver.blackbox + + @blackbox.setter + def blackbox(self, value): + """ + Set the BlackboxFunction wrapper class encapsulating the loss function or a function accepting a hyperparameter set + and returning a float. + + :return: + """ + self._solver.blackbox = value + + def get_results(self): + """ + Just call get_results of the member solver and return the result. + :return: return value of self._solver.get_results() + """ + # Only rank==0 returns results, the workers return None. + mpi_rank = self._mpi_comm.Get_rank() + if mpi_rank == 0: + return self._solver.get_results() + return None, None + + def run_worker_mode(self): + """ + This function is called if the wrapper should run as a worker for a specific MPI rank. + It receives messages for the following tags: + tag==MPI_SEND_CANDIDATE: parameters for the loss calculation. It param==None, the worker finishes. + It sends messages for the following tags: + tag==MPI_SEND_RESULT: result of an evaluated candidate. + + :return: the evaluated loss of the candidate + """ + rank = self._mpi_comm.Get_rank() + print("Starting worker {}. Waiting for param...".format(rank)) + + cand_results = dict() + + while True: + try: + candidate = self._mpi_comm.recv(source=0, tag=MPI_TAGS.MPI_SEND_CANDIDATE.value) # Wait here till params are received + + if candidate is None: + print("[RECEIVE] Process {} received finish signal.".format(rank)) + return + + # if candidate.ID == 9999: + # comm.gather(losses, root=0) + # continue + + # print("[WORKING] Process {} is actually doing things.".format(rank)) + cand_id = candidate.ID + params = candidate.get_values() + + try: + loss = self._solver.blackbox.blackbox_func(params) + except: + loss = self._solver.blackbox.blackbox_func(**params) + + except Exception as e: + msg = "Error in Worker(rank={}): {}".format(rank, e) + LOG.error(msg) + print(msg) + + loss = np.nan + finally: + cand_results['book_time'] = datetime.datetime.now() + cand_results['loss'] = loss # Write loss to dictionary. This dictionary will be send back to the master via gather + cand_results['refresh_time'] = datetime.datetime.now() + + cand_results['book_time'] = datetime.datetime.now() + + cand_results['loss'] = loss # Write loss to dictionary. This dictionary will be send back to the master via gather + cand_results['refresh_time'] = datetime.datetime.now() + + self._mpi_comm.send((cand_id, cand_results), dest=0, tag=MPI_TAGS.MPI_SEND_RESULTS.value) + + def signal_worker_finished(self): + """ + This function sends data==None to all workers from the master. This is the signal that tells the workers to finish. + + :return: + """ + print('[SEND] signal_worker_finished') + size = self._mpi_comm.Get_size() + for i in range(size - 1): + self._mpi_comm.send(None, dest=i + 1, tag=MPI_TAGS.MPI_SEND_CANDIDATE.value) + + def run(self, *args, **kwargs): + """ + This function starts the optimization process of the underlying solver and takes care of the MPI awareness. + """ + + mpi_rank = self._mpi_comm.Get_rank() + if mpi_rank == 0: + # This is the master process. From here we run the solver and start all the other processes. + self._solver.run(*args, **kwargs) + self.signal_worker_finished() # Tell the workers to finish. + else: + # this script execution should be in worker mode as it is an mpi worker. + self.run_worker_mode() + + def is_master(self): + mpi_rank = self._mpi_comm.Get_rank() + if mpi_rank == 0: + return True + else: + return False + + def is_worker(self): + mpi_rank = self._mpi_comm.Get_rank() + if mpi_rank != 0: + return True + else: + return False diff --git a/hyppopy/solvers/OptunitySolver.py b/hyppopy/solvers/OptunitySolver.py index 8a7abe7..858af11 100644 --- a/hyppopy/solvers/OptunitySolver.py +++ b/hyppopy/solvers/OptunitySolver.py @@ -1,137 +1,203 @@ # Hyppopy - A Hyper-Parameter Optimization Toolbox # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE import os import logging import optunity from pprint import pformat + +from hyppopy.CandidateDescriptor import CandidateDescriptor, CandicateDescriptorWrapper from hyppopy.globals import DEBUGLEVEL LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) from hyppopy.solvers.HyppopySolver import HyppopySolver class OptunitySolver(HyppopySolver): def __init__(self, project=None): """ The constructor accepts a HyppopyProject. :param project: [HyppopyProject] project instance, default=None """ HyppopySolver.__init__(self, project) def define_interface(self): """ This function is called when HyppopySolver.__init__ function finished. Child classes need to define their individual parameter here by calling the _add_member function for each class member variable need to be defined. Using _add_hyperparameter_signature the structure of a hyperparameter the solver expects must be defined. Both, members and hyperparameter signatures are later get checked, before executing the solver, ensuring settings passed fullfill solver needs. """ self._add_member("max_iterations", int) self._add_hyperparameter_signature(name="domain", dtype=str, options=["uniform", "categorical"]) self._add_hyperparameter_signature(name="data", dtype=list) self._add_hyperparameter_signature(name="type", dtype=type) def loss_function_call(self, params): """ This function is called within the function loss_function and encapsulates the actual blackbox function call in each iteration. The function loss_function takes care of the iteration driving and reporting, but each solver lib might need some special treatment between the parameter set selection and the calling of the actual blackbox function, e.g. parameter converting. :param params: [dict] hyperparameter space sample e.g. {'p1': 0.123, 'p2': 3.87, ...} :return: [float] loss """ for key in params.keys(): if self.project.get_typeof(key) is int: params[key] = int(round(params[key])) return self.blackbox(**params) + def loss_function_batch(self, **candidates): + """ + This function is called with a list of candidates. This list is driven by the solver lib itself. + The purpose of this function is to take care of the iteration reporting and the calling + of the callback_func if available. As a developer you might want to overwrite this function (or the 'non-batch'-version completely (e.g. + HyperoptSolver) but then you need to take care for iteration reporting for yourself. The alternative is to only + implement loss_function_call (e.g. OptunitySolver). + + :param candidates: [list of CandidateDescriptors] + + :return: [dict] result e.g. {'loss': 0.5, 'book_time': ..., 'refresh_time': ...} + """ + + candidate_list = [] + + keysValue = candidates.keys() + temp = {} + for key in keysValue: + temp[key] = candidates[key].get() + + for i, pack in enumerate(zip(*temp.values())): + candidate_list.append(CandidateDescriptor(**(dict(zip(keysValue, pack))))) + + results = super(OptunitySolver, self).loss_function_batch(candidate_list) + try: + self.best = self._trials.argmin + except: + pass + + result = [x['loss'] for x in results.values()] + return result + + def hyppopy_optunity_solver_pmap(self, f, seq): + # Check if seq is empty. I so, return an empty result list. + if len(seq) == 0: + return [] + + candidates = [] + for elem in seq: + can = CandidateDescriptor(**elem) + candidates.append(can) + + cand_list = CandicateDescriptorWrapper(keys=seq[0].keys()) + cand_list.set(candidates) + + f_result = f(cand_list) + + # If one candidate does not match the constraints, f() returns a single default value. + # This is a problem as all the other candidates are not calculated either. + # The following is a workaround. We split the candidate_list into 2 lists and call the map function recursively until all valid parameters are processed. + if not isinstance(f_result, list): + # First half + seq_A = seq[:len(seq) // 2] + temp_result_a = self.hyppopy_optunity_solver_pmap(f, seq_A) + + seq_B = seq[len(seq) // 2:] + temp_result_b = self.hyppopy_optunity_solver_pmap(f, seq_B) + # f_result = [42] + + f_result = temp_result_a + temp_result_b + + return f_result + def execute_solver(self, searchspace): """ This function is called immediately after convert_searchspace and get the output of the latter as input. It's purpose is to call the solver libs main optimization function. :param searchspace: converted hyperparameter space """ LOG.debug("execute_solver using solution space:\n\n\t{}\n".format(pformat(searchspace))) try: - self.best, _, _ = optunity.minimize_structured(f=self.loss_function, - num_evals=self.max_iterations, - search_space=searchspace) + optunity.minimize_structured(f=self.loss_function_batch, + num_evals=self.max_iterations, + search_space=searchspace, + pmap=self.hyppopy_optunity_solver_pmap) except Exception as e: LOG.error("internal error in optunity.minimize_structured occured. {}".format(e)) raise BrokenPipeError("internal error in optunity.minimize_structured occured. {}".format(e)) def split_categorical(self, pdict): """ This function splits the incoming dict into two parts, categorical only entries and other. :param pdict: [dict] input parameter description dict :return: [dict],[dict] categorical only, others """ categorical = {} uniform = {} for name, pset in pdict.items(): for key, value in pset.items(): if key == 'domain' and value == 'categorical': categorical[name] = pset elif key == 'domain': uniform[name] = pset return categorical, uniform def convert_searchspace(self, hyperparameter): """ This function gets the unified hyppopy-like parameterspace description as input and, if necessary, should convert it into a solver lib specific format. The function is invoked when run is called and what it returns is passed as searchspace argument to the function execute_solver. :param hyperparameter: [dict] nested parameter description dict e.g. {'name': {'domain':'uniform', 'data':[0,1], 'type':'float'}, ...} :return: [object] converted hyperparameter space """ LOG.debug("convert input parameter\n\n\t{}\n".format(pformat(hyperparameter))) # split input in categorical and non-categorical data cat, uni = self.split_categorical(hyperparameter) # build up dictionary keeping all non-categorical data uniforms = {} for key, value in uni.items(): for key2, value2 in value.items(): if key2 == 'data': if len(value2) == 3: uniforms[key] = value2[0:2] elif len(value2) == 2: uniforms[key] = value2 else: raise AssertionError("precondition violation, optunity searchspace needs list with left and right range bounds!") if len(cat) == 0: return uniforms # build nested categorical structure inner_level = uniforms for key, value in cat.items(): tmp = {} optunity_space = {} for key2, value2 in value.items(): if key2 == 'data': for elem in value2: tmp[elem] = inner_level optunity_space[key] = tmp inner_level = optunity_space return optunity_space diff --git a/hyppopy/solvers/RandomsearchSolver.py b/hyppopy/solvers/RandomsearchSolver.py index 23a1a62..60cab3d 100644 --- a/hyppopy/solvers/RandomsearchSolver.py +++ b/hyppopy/solvers/RandomsearchSolver.py @@ -1,209 +1,207 @@ # Hyppopy - A Hyper-Parameter Optimization Toolbox # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE +from hyppopy.CandidateDescriptor import CandidateDescriptor __all__ = ['RandomsearchSolver', 'draw_uniform_sample', 'draw_normal_sample', 'draw_loguniform_sample', 'draw_categorical_sample', 'draw_sample'] import os import copy import random import logging import numpy as np from pprint import pformat from hyppopy.globals import DEBUGLEVEL from hyppopy.solvers.HyppopySolver import HyppopySolver LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) def draw_uniform_sample(param): """ Function draws a random sample from a uniform range :param param: [dict] input hyperparameter discription :return: random sample value of type data['type'] """ assert param['type'] is not str, "cannot sample a string list!" assert param['data'][0] < param['data'][1], "precondition violation: data[0] > data[1]!" s = random.random() s *= np.abs(param['data'][1] - param['data'][0]) s += param['data'][0] if param['type'] is int: s = int(np.round(s)) if s < param['data'][0]: s = int(param['data'][0]) if s > param['data'][1]: s = int(param['data'][1]) return s def draw_normal_sample(param): """ Function draws a random sample from a normal distributed range :param param: [dict] input hyperparameter discription :return: random sample value of type data['type'] """ assert param['type'] is not str, "cannot sample a string list!" assert param['data'][0] < param['data'][1], "precondition violation: data[0] > data[1]!" mu = (param['data'][1] - param['data'][0]) / 2 sigma = mu / 3 s = np.random.normal(loc=param['data'][0] + mu, scale=sigma) if s > param['data'][1]: s = param['data'][1] if s < param['data'][0]: s = param['data'][0] s = float(s) if param["type"] is int: s = int(np.round(s)) return s def draw_loguniform_sample(param): """ Function draws a random sample from a logarithmic distributed range :param param: [dict] input hyperparameter discription :return: random sample value of type data['type'] """ assert param['type'] is not str, "cannot sample a string list!" assert param['data'][0] < param['data'][1], "precondition violation: data[0] > data[1]!" p = copy.deepcopy(param) p['data'][0] = np.log(param['data'][0]) p['data'][1] = np.log(param['data'][1]) assert p['data'][0] is not np.nan, "Precondition violation, left bound input error, results in nan!" assert p['data'][1] is not np.nan, "Precondition violation, right bound input error, results in nan!" x = draw_uniform_sample(p) s = np.exp(x) if s > param['data'][1]: s = param['data'][1] if s < param['data'][0]: s = param['data'][0] return s def draw_categorical_sample(param): """ Function draws a random sample from a categorical list :param param: [dict] input hyperparameter discription :return: random sample value of type data['type'] """ return random.sample(param['data'], 1)[0] def draw_sample(param): """ Function draws a sample from the input hyperparameter descriptor depending on it's domain :param param: [dict] input hyperparameter discription :return: random sample value of type data['type'] """ assert isinstance(param, dict), "input error, hyperparam descriptors of type {} not allowed!".format(type(param)) if param['domain'] == "uniform": return draw_uniform_sample(param) elif param['domain'] == "normal": return draw_normal_sample(param) elif param['domain'] == "loguniform": return draw_loguniform_sample(param) elif param['domain'] == "categorical": return draw_categorical_sample(param) else: raise LookupError("Unknown domain {}".format(param['domain'])) class RandomsearchSolver(HyppopySolver): """ The RandomsearchSolver class implements a randomsearch optimization. The randomsearch supports categorical, uniform, normal and loguniform sampling. The solver draws an independent sample from the parameter space each iteration. """ def __init__(self, project=None): """ The constructor accepts a HyppopyProject. :param project: [HyppopyProject] project instance, default=None """ HyppopySolver.__init__(self, project) def define_interface(self): """ This function is called when HyppopySolver.__init__ function finished. Child classes need to define their individual parameter here by calling the _add_member function for each class member variable need to be defined. Using _add_hyperparameter_signature the structure of a hyperparameter the solver expects must be defined. Both, members and hyperparameter signatures are later get checked, before executing the solver, ensuring settings passed fullfill solver needs. """ self._add_member("max_iterations", int) self._add_hyperparameter_signature(name="domain", dtype=str, options=["uniform", "normal", "loguniform", "categorical"]) self._add_hyperparameter_signature(name="data", dtype=list) self._add_hyperparameter_signature(name="type", dtype=type) - def loss_function_call(self, params): + def get_candidates(self, searchspace): """ - This function is called within the function loss_function and encapsulates the actual blackbox function call - in each iteration. The function loss_function takes care of the iteration driving and reporting, but each solver - lib might need some special treatment between the parameter set selection and the calling of the actual blackbox - function, e.g. parameter converting. + This function converts the searchspace to a candidate_list that can then be used to distribute via MPI. - :param params: [dict] hyperparameter space sample e.g. {'p1': 0.123, 'p2': 3.87, ...} - - :return: [float] loss + :param searchspace: converted hyperparameter space """ - loss = self.blackbox(**params) - if loss is None: - return np.nan - return loss + candidates_list = list() + N = self.max_iterations + for n in range(N): + params = {} + for name, p in searchspace.items(): + params[name] = draw_sample(p) + candidates_list.append(CandidateDescriptor(**params)) + + return candidates_list def execute_solver(self, searchspace): """ This function is called immediately after convert_searchspace and get the output of the latter as input. It's purpose is to call the solver libs main optimization function. :param searchspace: converted hyperparameter space """ - N = self.max_iterations + + candidates = self.get_candidates(searchspace) try: - for n in range(N): - params = {} - for name, p in searchspace.items(): - params[name] = draw_sample(p) - self.loss_function(**params) + self.loss_function_batch(candidates) except Exception as e: msg = "internal error in randomsearch execute_solver occured. {}".format(e) LOG.error(msg) raise BrokenPipeError(msg) self.best = self._trials.argmin def convert_searchspace(self, hyperparameter): """ This function gets the unified hyppopy-like parameterspace description as input and, if necessary, should convert it into a solver lib specific format. The function is invoked when run is called and what it returns is passed as searchspace argument to the function execute_solver. :param hyperparameter: [dict] nested parameter description dict e.g. {'name': {'domain':'uniform', 'data':[0,1], 'type':'float'}, ...} :return: [object] converted hyperparameter space """ LOG.debug("convert input parameter\n\n\t{}\n".format(pformat(hyperparameter))) return hyperparameter diff --git a/mpiplayground.py b/mpiplayground.py new file mode 100644 index 0000000..688434d --- /dev/null +++ b/mpiplayground.py @@ -0,0 +1,90 @@ +# DKFZ +# +# +# Copyright (c) German Cancer Research Center, +# Division of Medical Image Computing. +# All rights reserved. +# +# This software is distributed WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. +# +# See LICENSE + +# A hyppopy minimal example optimizing a simple demo function f(x,y) = x**2+y**2 + +# import the HyppopyProject class keeping track of inputs +from mpi4py import MPI + +from hyppopy.BlackboxFunction import BlackboxFunction +from hyppopy.HyppopyProject import HyppopyProject +from hyppopy.SolverPool import SolverPool +from hyppopy.solvers.MPISolverWrapper import MPISolverWrapper +from hyppopy.MPIBlackboxFunction import MPIBlackboxFunction + +# To configure the Hyppopy solver we use a simple nested dictionary with two obligatory main sections, +# hyperparameter and settings. The hyperparameter section defines your searchspace. Each hyperparameter +# is again a dictionary with: +# +# - a domain ['categorical', 'uniform', 'normal', 'loguniform'] +# - the domain data [left bound, right bound] and +# - a type of your domain ['str', 'int', 'float'] +# +# The settings section has two subcategories, solver and custom. The first contains settings for the solver, +# here 'max_iterations' - is the maximum number of iteration. +# +# The custom section allows defining custom parameter. An entry here is transformed to a member variable of the +# HyppopyProject class. These can be useful when implementing new solver classes or for control your hyppopy script. +# Here we use it as a solver switch to control the usage of our solver via the config. This means with the script +# below your can try out every solver by changing use_solver to 'optunity', 'randomsearch', 'gridsearch',... +# It can be used like so: project.custom_use_plugin (see below) If using the gridsearch solver, max_iterations is +# ignored, instead each hyperparameter must specifiy a number of samples additionally to the range like so: +# 'data': [0, 1, 100] which means sampling the space from 0 to 1 in 100 intervals. + +config = { + "hyperparameter": { + "x": { + "domain": "uniform", + "data": [-10.0, 10.0], + "type": float, + "frequency": 10 + }, + "y": { + "domain": "uniform", + "data": [-10.0, 10.0], + "type": float, + "frequency": 10 + } + }, + "max_iterations": 500, + "solver": "optunity" +} + +project = HyppopyProject(config=config) + + +# The user defined loss function +def my_loss_function_params(params): + x = params['x'] + y = params['y'] + return x**2+y**3 + +# The user defined loss function +def my_loss_function(x, y): + return x**2+y**3 + + +comm = MPI.COMM_WORLD +solver = MPISolverWrapper(solver=SolverPool.get(project=project), mpi_comm=comm) +blackbox = MPIBlackboxFunction(blackbox_func=my_loss_function_params, mpi_comm=comm) +solver.blackbox = blackbox + +solver.run() + +df, best = solver.get_results() + +if solver.is_master() is True: + print("\n") + print("*" * 100) + print("Best Parameter Set:\n{}".format(best)) + print("*" * 100) diff --git a/mpiplayground_dynpso.py b/mpiplayground_dynpso.py new file mode 100644 index 0000000..71511e8 --- /dev/null +++ b/mpiplayground_dynpso.py @@ -0,0 +1,54 @@ +# Minimal setup to test dynPSO code: Reproduce normal PSO with dynamic PSO. + +# Insert path to Marie's optunity +import sys + +dir = None +assert dir != None, 'Please adapt the path to the location of specialized Optunity' +sys.path.insert(1, dir) + + +from mpi4py import MPI + +from hyppopy.MPIBlackboxFunction import MPIBlackboxFunction +from hyppopy.solvers.MPISolverWrapper import MPISolverWrapper + +import hyppopy.HyppopyProject +import hyppopy.solvers.DynamicPSOSolver +import numpy + + +def updateParam(pop_history, num_params_obj): + return numpy.ones(num_params_obj) + + +def combineObj(args, params): + return sum([a * p for a, p in zip(args, params)]) + + +def f(x, y): + return [x ** 2, y ** 2] + + +project = hyppopy.HyppopyProject.HyppopyProject() +project.add_hyperparameter(name="x", domain="uniform", data=[-10, 10], type=float) +project.add_hyperparameter(name="y", domain="uniform", data=[-10, 10], type=float) +project.add_setting(name="max_iterations", value=300) +project.add_setting(name="num_params_obj", value=2) +project.add_setting(name="num_args_obj", value=2) +project.add_setting(name="combine_obj", value=combineObj) +project.add_setting(name="update_param", value=updateParam) +project.add_setting(name="phi1", value=1.5) +project.add_setting(name="phi2", value=2.0) + +# ====================================================================================== +my_solver = hyppopy.solvers.DynamicPSOSolver.DynamicPSOSolver(project) +# solver.blackbox = f + +comm = MPI.COMM_WORLD +solver = MPISolverWrapper(solver=my_solver, mpi_comm=comm) +blackbox = MPIBlackboxFunction(blackbox_func=f, mpi_comm=comm) +solver.blackbox = blackbox +# ====================================================================================== + +solver.run() \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 2c5bae8..a5ec293 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,13 @@ hyperopt>=0.1.2 matplotlib>=3.0.3 +mpi4py==3.0.2 numpy>=1.16.2 optuna>=0.9.0 Optunity>=1.1.1 pandas>=0.24.2 pytest>=4.3.1 scikit-learn>=0.20.3 scipy>=1.2.1 visdom>=0.1.8.8 xmlrunner>=1.7.7 Sphinx>=1.8.3 \ No newline at end of file