diff --git a/hyppopy/BlackboxFunction.py b/hyppopy/BlackboxFunction.py index ce5fde6..1e24f40 100644 --- a/hyppopy/BlackboxFunction.py +++ b/hyppopy/BlackboxFunction.py @@ -1,183 +1,182 @@ # Hyppopy - A Hyper-Parameter Optimization Toolbox # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE __all__ = ['BlackboxFunction'] import os import logging import functools from hyppopy.globals import DEBUGLEVEL LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) def default_kwargs(**defaultKwargs): """ Decorator defining default args in **kwargs arguments """ def actual_decorator(fn): @functools.wraps(fn) def g(*args, **kwargs): defaultKwargs.update(kwargs) return fn(*args, **defaultKwargs) return g return actual_decorator class BlackboxFunction(object): """ This class is a BlackboxFunction wrapper class encapsulating the loss function. Additional function pointer can be set to get access at different pipelining steps: - dataloader_func: data loading, the function must return a data object and is called first when the solver is executed. The data object returned will be the input of the blackbox function. - preprocess_func: data preprocessing is called after dataloader_func, the functions signature must be foo(data, params) and must return a data object. The input is the data object set directly or via dataloader_func, the params are passed from constructor params. - callback_func: this function is called at each iteration step getting passed the trail info content, can be used for custom visualization - data: add a data object directly The constructor accepts several function pointers or a data object which are all None by default (see below). Additionally one can define an arbitrary number of arg pairs. These are passed as input to each function pointer as arguments. :param dataloader_func: data loading function pointer, default=None :param preprocess_func: data preprocessing function pointer, default=None :param callback_func: callback function pointer, default=None :param data: data object, default=None :param kwargs: additional arg=value pairs """ @default_kwargs(blackbox_func=None, dataloader_func=None, preprocess_func=None, callback_func=None, data=None) def __init__(self, **kwargs): self._blackbox_func = None self._preprocess_func = None self._dataloader_func = None self._callback_func = None self._raw_data = None self._data = None self.setup(kwargs) def __call__(self, **kwargs): """ Call method calls blackbox_func passing the data object and the args passed :param kwargs: [dict] args :return: blackbox_func(data, kwargs) """ try: try: return self.blackbox_func(self.data, kwargs) except: return self.blackbox_func(self.data, **kwargs) except: try: return self.blackbox_func(kwargs) except: return self.blackbox_func(**kwargs) def setup(self, kwargs): """ Alternative to Constructor, kwargs signature see __init__ :param kwargs: (see __init__) """ self._blackbox_func = kwargs['blackbox_func'] self._preprocess_func = kwargs['preprocess_func'] self._dataloader_func = kwargs['dataloader_func'] self._callback_func = kwargs['callback_func'] self._raw_data = kwargs['data'] self._data = self._raw_data del kwargs['blackbox_func'] del kwargs['preprocess_func'] del kwargs['dataloader_func'] del kwargs['data'] params = kwargs if self.dataloader_func is not None: self._raw_data = self.dataloader_func(params=params) - # assert self._raw_data is not None, "Missing data exception!" assert self.blackbox_func is not None, "Missing blackbox function exception!" if self.preprocess_func is not None: result = self.preprocess_func(data=self._raw_data, params=params) if result is not None: self._data = result else: self._data = self._raw_data else: self._data = self._raw_data @property def blackbox_func(self): """ BlackboxFunction wrapper class encapsulating the loss function or a function accepting a hyperparameter set and returning a float. :return: [object] pointer to blackbox_func """ return self._blackbox_func @property def preprocess_func(self): """ Data preprocessing is called after dataloader_func, the functions signature must be foo(data, params) and must return a data object. The input is the data object set directly or via dataloader_func, the params are passed from constructor params. :return: [object] preprocess_func """ return self._preprocess_func @property def dataloader_func(self): """ Data loading, the function must return a data object and is called first when the solver is executed. The data object returned will be the input of the blackbox function. :return: [object] dataloader_func """ return self._dataloader_func @property def callback_func(self): """ This function is called at each iteration step getting passed the trail info content, can be used for custom visualization :return: [object] callback_func """ return self._callback_func @property def raw_data(self): """ This data structure is used to store the return from dataloader_func to serve as input for preprocess_func if available. :return: [object] raw_data """ return self._raw_data @property def data(self): """ Datastructure keeping the input data. :return: [object] data """ return self._data diff --git a/hyppopy/solvers/MPISolverWrapper.py b/hyppopy/solvers/MPISolverWrapper.py index e95b841..a0a678f 100644 --- a/hyppopy/solvers/MPISolverWrapper.py +++ b/hyppopy/solvers/MPISolverWrapper.py @@ -1,166 +1,171 @@ # Hyppopy - A Hyper-Parameter Optimization Toolbox # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE import datetime import os import logging import numpy as np from mpi4py import MPI from hyppopy.globals import DEBUGLEVEL, MPI_TAGS +from hyppopy.MPIBlackboxFunction import MPIBlackboxFunction LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) class MPISolverWrapper: """ TODO Class description The MPISolverWrapper class wraps the functionality of solvers in Hyppopy to extend them with MPI functionality. It builds upon the interface defined by the HyppopySolver class. """ def __init__(self, solver=None, mpi_comm=None): """ The constructor accepts a HyppopySolver. :param solver: [HyppopySolver] solver instance, default=None :param mpi_comm: [MPI communicator] MPI communicator instance. If None, we create a new MPI.COMM_WORLD, default=None """ self._solver = solver self._mpi_comm = None if mpi_comm is None: print('MPISolverWrapper: No mpi_comm given: Using MPI.COMM_WORLD') self._mpi_comm = MPI.COMM_WORLD else: self._mpi_comm = mpi_comm @property def blackbox(self): """ Get the BlackboxFunction object. :return: [object] BlackboxFunction instance or function of member solver """ return self._solver.blackbox @blackbox.setter def blackbox(self, value): """ - Set the BlackboxFunction wrapper class encapsulating the loss function or a function accepting a hyperparameter set - and returning a float. - + Set the BlackboxFunction wrapper class encapsulating the loss function or a function accepting a hyperparameter + set and returning a float. + If the passed value is not an instance of MPIBlackboxFunction (or a derived class) it will automatically + wrapped by an MPIBackboxFunction. :return: """ - self._solver.blackbox = value + if isinstance(value, MPIBlackboxFunction): + self._solver.blackbox = value + else: + self._solver.blackbox = MPIBlackboxFunction(blackbox_func=value, mpi_comm=self._mpi_comm) def get_results(self): """ Just call get_results of the member solver and return the result. :return: return value of self._solver.get_results() """ # Only rank==0 returns results, the workers return None. mpi_rank = self._mpi_comm.Get_rank() if mpi_rank == 0: return self._solver.get_results() return None, None def run_worker_mode(self): """ This function is called if the wrapper should run as a worker for a specific MPI rank. It receives messages for the following tags: tag==MPI_SEND_CANDIDATE: parameters for the loss calculation. It param==None, the worker finishes. It sends messages for the following tags: tag==MPI_SEND_RESULT: result of an evaluated candidate. :return: the evaluated loss of the candidate """ rank = self._mpi_comm.Get_rank() print("Starting worker {}. Waiting for param...".format(rank)) cand_results = dict() while True: try: candidate = self._mpi_comm.recv(source=0, tag=MPI_TAGS.MPI_SEND_CANDIDATE.value) # Wait here till params are received if candidate is None: print("[RECEIVE] Process {} received finish signal.".format(rank)) return # if candidate.ID == 9999: # comm.gather(losses, root=0) # continue # print("[WORKING] Process {} is actually doing things.".format(rank)) cand_id = candidate.ID params = candidate.get_values() try: loss = self._solver.blackbox.blackbox_func(params) except: loss = self._solver.blackbox.blackbox_func(**params) except Exception as e: msg = "Error in Worker(rank={}): {}".format(rank, e) LOG.error(msg) print(msg) loss = np.nan finally: cand_results['book_time'] = datetime.datetime.now() cand_results['loss'] = loss # Write loss to dictionary. This dictionary will be send back to the master via gather cand_results['refresh_time'] = datetime.datetime.now() cand_results['book_time'] = datetime.datetime.now() cand_results['loss'] = loss # Write loss to dictionary. This dictionary will be send back to the master via gather cand_results['refresh_time'] = datetime.datetime.now() self._mpi_comm.send((cand_id, cand_results), dest=0, tag=MPI_TAGS.MPI_SEND_RESULTS.value) def signal_worker_finished(self): """ This function sends data==None to all workers from the master. This is the signal that tells the workers to finish. :return: """ print('[SEND] signal_worker_finished') size = self._mpi_comm.Get_size() for i in range(size - 1): self._mpi_comm.send(None, dest=i + 1, tag=MPI_TAGS.MPI_SEND_CANDIDATE.value) def run(self, *args, **kwargs): """ This function starts the optimization process of the underlying solver and takes care of the MPI awareness. """ mpi_rank = self._mpi_comm.Get_rank() if mpi_rank == 0: # This is the master process. From here we run the solver and start all the other processes. self._solver.run(*args, **kwargs) self.signal_worker_finished() # Tell the workers to finish. else: # this script execution should be in worker mode as it is an mpi worker. self.run_worker_mode() def is_master(self): mpi_rank = self._mpi_comm.Get_rank() if mpi_rank == 0: return True else: return False def is_worker(self): mpi_rank = self._mpi_comm.Get_rank() if mpi_rank != 0: return True else: return False diff --git a/mpiplayground.py b/mpiplayground.py index 688434d..185e5c9 100644 --- a/mpiplayground.py +++ b/mpiplayground.py @@ -1,90 +1,78 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE # A hyppopy minimal example optimizing a simple demo function f(x,y) = x**2+y**2 # import the HyppopyProject class keeping track of inputs -from mpi4py import MPI - -from hyppopy.BlackboxFunction import BlackboxFunction from hyppopy.HyppopyProject import HyppopyProject from hyppopy.SolverPool import SolverPool from hyppopy.solvers.MPISolverWrapper import MPISolverWrapper -from hyppopy.MPIBlackboxFunction import MPIBlackboxFunction # To configure the Hyppopy solver we use a simple nested dictionary with two obligatory main sections, # hyperparameter and settings. The hyperparameter section defines your searchspace. Each hyperparameter # is again a dictionary with: # # - a domain ['categorical', 'uniform', 'normal', 'loguniform'] # - the domain data [left bound, right bound] and # - a type of your domain ['str', 'int', 'float'] # # The settings section has two subcategories, solver and custom. The first contains settings for the solver, # here 'max_iterations' - is the maximum number of iteration. # # The custom section allows defining custom parameter. An entry here is transformed to a member variable of the # HyppopyProject class. These can be useful when implementing new solver classes or for control your hyppopy script. # Here we use it as a solver switch to control the usage of our solver via the config. This means with the script # below your can try out every solver by changing use_solver to 'optunity', 'randomsearch', 'gridsearch',... # It can be used like so: project.custom_use_plugin (see below) If using the gridsearch solver, max_iterations is # ignored, instead each hyperparameter must specifiy a number of samples additionally to the range like so: # 'data': [0, 1, 100] which means sampling the space from 0 to 1 in 100 intervals. config = { "hyperparameter": { "x": { "domain": "uniform", "data": [-10.0, 10.0], "type": float, "frequency": 10 }, "y": { "domain": "uniform", "data": [-10.0, 10.0], "type": float, "frequency": 10 } }, "max_iterations": 500, "solver": "optunity" } project = HyppopyProject(config=config) - # The user defined loss function def my_loss_function_params(params): x = params['x'] y = params['y'] return x**2+y**3 -# The user defined loss function -def my_loss_function(x, y): - return x**2+y**3 - - -comm = MPI.COMM_WORLD -solver = MPISolverWrapper(solver=SolverPool.get(project=project), mpi_comm=comm) -blackbox = MPIBlackboxFunction(blackbox_func=my_loss_function_params, mpi_comm=comm) -solver.blackbox = blackbox +solver = MPISolverWrapper(solver=SolverPool.get(project=project)) +solver.blackbox = my_loss_function_params solver.run() df, best = solver.get_results() if solver.is_master() is True: print("\n") print("*" * 100) print("Best Parameter Set:\n{}".format(best)) print("*" * 100)