diff --git a/hyppopy/BlackboxFunction.py b/hyppopy/BlackboxFunction.py index 32cce46..729d984 100644 --- a/hyppopy/BlackboxFunction.py +++ b/hyppopy/BlackboxFunction.py @@ -1,135 +1,171 @@ # Hyppopy - A Hyper-Parameter Optimization Toolbox # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE __all__ = ['BlackboxFunction'] import os import logging import functools from hyppopy.globals import DEBUGLEVEL LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) def default_kwargs(**defaultKwargs): """ Decorator defining default args in **kwargs arguments """ def actual_decorator(fn): @functools.wraps(fn) def g(*args, **kwargs): defaultKwargs.update(kwargs) return fn(*args, **defaultKwargs) return g return actual_decorator class BlackboxFunction(object): """ This class is a BlackboxFunction wrapper class encapsulating the loss function. Additional function pointer can be set to get access at different pipelining steps: - dataloader_func: data loading, the function must return a data object and is called first when the solver is executed. The data object returned will be the input of the blackbox function. - preprocess_func: data preprocessing is called after dataloader_func, the functions signature must be foo(data, params) and must return a data object. The input is the data object set directly or via dataloader_func, the params are passed from constructor params. - callback_func: this function is called at each iteration step getting passed the trail info content, can be used for custom visualization - data: add a data object directly + + The constructor accepts several function pointers or a data object which are all None by default (see below). + Additionally one can define an arbitrary number of arg pairs. These are passed as input to each function pointer as + arguments. + + :param dataloader_func: data loading function pointer, default=None + :param preprocess_func: data preprocessing function pointer, default=None + :param callback_func: callback function pointer, default=None + :param data: data object, default=None + :param kwargs: additional arg=value pairs """ @default_kwargs(blackbox_func=None, dataloader_func=None, preprocess_func=None, callback_func=None, data=None) def __init__(self, **kwargs): - """ - Constructor accepts function pointer or a data object which are all None by default. Additionally one can define - an arbitrary number of arg pairs. These are passed as input to each function pointer as arguments. - - :param dataloader_func: data loading function pointer, default=None - :param preprocess_func: data preprocessing function pointer, default=None - :param callback_func: callback function pointer, default=None - :param data: data object, default=None - :param kwargs: additional arg=value pairs - """ self._blackbox_func = None self._preprocess_func = None self._dataloader_func = None self._callback_func = None self._raw_data = None self._data = None self.setup(kwargs) def __call__(self, **kwargs): """ Call method calls blackbox_func passing the data object and the args passed :param kwargs: [dict] args :return: blackbox_func(data, kwargs) """ return self.blackbox_func(self.data, kwargs) def setup(self, kwargs): """ Alternative to Constructor, kwargs signature see __init__ :param kwargs: (see __init__) """ self._blackbox_func = kwargs['blackbox_func'] self._preprocess_func = kwargs['preprocess_func'] self._dataloader_func = kwargs['dataloader_func'] self._callback_func = kwargs['callback_func'] self._raw_data = kwargs['data'] self._data = self._raw_data del kwargs['blackbox_func'] del kwargs['preprocess_func'] del kwargs['dataloader_func'] del kwargs['data'] params = kwargs if self.dataloader_func is not None: self._raw_data = self.dataloader_func(params=params) assert self._raw_data is not None, "Missing data exception!" assert self.blackbox_func is not None, "Missing blackbox fucntion exception!" if self.preprocess_func is not None: result = self.preprocess_func(data=self._raw_data, params=params) if result is not None: self._data = result else: self._data = self._raw_data else: self._data = self._raw_data @property def blackbox_func(self): + """ + BlackboxFunction wrapper class encapsulating the loss function or a function accepting a hyperparameter set and + returning a float. + + :return: [object] pointer to blackbox_func + """ return self._blackbox_func @property def preprocess_func(self): + """ + Data preprocessing is called after dataloader_func, the functions signature must be foo(data, params) and must + return a data object. The input is the data object set directly or via dataloader_func, the params are passed + from constructor params. + + :return: [object] preprocess_func + """ return self._preprocess_func @property def dataloader_func(self): + """ + Data loading, the function must return a data object and is called first when the solver is executed. The data + object returned will be the input of the blackbox function. + + :return: [object] dataloader_func + """ return self._dataloader_func @property def callback_func(self): + """ + This function is called at each iteration step getting passed the trail info content, can be used for + custom visualization + + :return: [object] callback_func + """ return self._callback_func @property def raw_data(self): + """ + This data structure is used to store the return from dataloader_func to serve as input for preprocess_func if + available. + + :return: [object] raw_data + """ return self._raw_data @property def data(self): + """ + Datastructure keeping the input data. + + :return: [object] data + """ return self._data diff --git a/hyppopy/SolverPool.py b/hyppopy/SolverPool.py index 71868bc..77a477b 100644 --- a/hyppopy/SolverPool.py +++ b/hyppopy/SolverPool.py @@ -1,96 +1,99 @@ # Hyppopy - A Hyper-Parameter Optimization Toolbox # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE __all__ = ['SolverPool'] from .Singleton import * import os import logging from hyppopy.HyppopyProject import HyppopyProject from hyppopy.solvers.OptunaSolver import OptunaSolver from hyppopy.solvers.HyperoptSolver import HyperoptSolver from hyppopy.solvers.OptunitySolver import OptunitySolver from hyppopy.solvers.GridsearchSolver import GridsearchSolver from hyppopy.solvers.RandomsearchSolver import RandomsearchSolver from hyppopy.solvers.QuasiRandomsearchSolver import QuasiRandomsearchSolver from hyppopy.globals import DEBUGLEVEL LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) @singleton_object class SolverPool(metaclass=Singleton): """ The SolverPool is a helper singleton class to get the desired solver either by name and a HyppopyProject instance or by a HyppopyProject instance only, if it defines a setting field called solver. """ def __init__(self): + """ + Constructor defines the solvers available. If a new solver should be added, add it's name to this list. + """ self._solver_list = ["hyperopt", "optunity", "optuna", "randomsearch", "quasirandomsearch", "gridsearch"] def get_solver_names(self): """ Returns a list of available solvers :return: [list] solver list """ return self._solver_list def get(self, solver_name=None, project=None): """ Get the configured solver instance :param solver_name: [str] solver name, if None, the project must have an attribute solver keeping the solver name, default=None :param project: [HyppopyProject] HyppopyProject instance :return: [HyppopySolver] the configured solver instance """ if solver_name is not None: assert isinstance(solver_name, str), "precondition violation, solver_name type str expected, got {} instead!".format(type(solver_name)) if project is not None: assert isinstance(project, HyppopyProject), "precondition violation, project type HyppopyProject expected, got {} instead!".format(type(project)) if "solver" in project.__dict__: solver_name = project.solver if solver_name not in self._solver_list: raise AssertionError("Solver named [{}] not implemented!".format(solver_name)) if solver_name == "hyperopt": if project is not None: return HyperoptSolver(project) return HyperoptSolver() elif solver_name == "optunity": if project is not None: return OptunitySolver(project) return OptunitySolver() elif solver_name == "optuna": if project is not None: return OptunaSolver(project) return OptunaSolver() elif solver_name == "gridsearch": if project is not None: return GridsearchSolver(project) return GridsearchSolver() elif solver_name == "randomsearch": if project is not None: return RandomsearchSolver(project) return RandomsearchSolver() elif solver_name == "quasirandomsearch": if project is not None: return QuasiRandomsearchSolver(project) return QuasiRandomsearchSolver() diff --git a/hyppopy/VisdomViewer.py b/hyppopy/VisdomViewer.py index d21ee51..bbb4e9c 100644 --- a/hyppopy/VisdomViewer.py +++ b/hyppopy/VisdomViewer.py @@ -1,160 +1,167 @@ # Hyppopy - A Hyper-Parameter Optimization Toolbox # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE __all__ = ['VisdomViewer'] import warnings import numpy as np from visdom import Visdom def time_formatter(time_s): """ Formats time in seconds input to more intuitive form h, min, s or ms, depending on magnitude :param time_s: [float] time in seconds :return: """ if time_s < 0.01: return int(time_s * 1000.0 * 1000) / 1000.0, "ms" elif 100 < time_s < 3600: return int(time_s / 60 * 1000) / 1000.0, "min" elif time_s >= 3600: return int(time_s / 3600 * 1000) / 1000.0, "h" else: return int(time_s * 1000) / 1000.0, "s" class VisdomViewer(object): """ The VisdomViewer class implements the live viewer plots via visdom. When extending implement your plot as methos and call it in update. Using this class make it necessary starting a visdom server beforehand $ python -m visdom.server """ def __init__(self, project, port=8097, server="http://localhost"): + """ + The constructor wants a HyppopyProject and accepts a visdom server port and a server name. + + :param project: [HyppopyProject] project instance + :param port: [int] server port, default=8097 + :param server: [str] server name, default=http://localhost + """ self._viz = Visdom(port=port, server=server) self._enabled = self._viz.check_connection(timeout_seconds=3) if not self._enabled: warnings.warn("No connection to visdom server established. Visualization cannot be displayed!") self._project = project self._best_win = None self._best_loss = None self._loss_iter_plot = None self._status_report = None self._axis_tags = None self._axis_plots = None def plot_losshistory(self, input_data): """ This function plots the loss history loss over iteration :param input_data: [dict] trail infos """ loss = np.array([input_data["loss"]]) iter = np.array([input_data["iterations"]]) if self._loss_iter_plot is None: self._loss_iter_plot = self._viz.line(loss, X=iter, opts=dict( markers=True, markersize=5, dash=np.array(['dashdot']), title="Loss History", xlabel='iteration', ylabel='loss' )) else: self._viz.line(loss, X=iter, win=self._loss_iter_plot, update='append') def plot_hyperparameter(self, input_data): """ This function plots each hyperparameter axis :param input_data: [dict] trail infos """ if self._axis_plots is None: self._axis_tags = [] self._axis_plots = {} for item in input_data.keys(): if item == "refresh_time" or item == "book_time" or item == "iterations" or item == "status" or item == "loss": continue self._axis_tags.append(item) for axis in self._axis_tags: xlabel = "value" if isinstance(input_data[axis], str): if self._project.hyperparameter[axis]["domain"] == "categorical": xlabel = '-'.join(self._project.hyperparameter[axis]["data"]) input_data[axis] = self._project.hyperparameter[axis]["data"].index(input_data[axis]) axis_loss = np.array([input_data[axis], input_data["loss"]]).reshape(1, -1) self._axis_plots[axis] = self._viz.scatter(axis_loss, opts=dict( markersize=5, title=axis, xlabel=xlabel, ylabel='loss')) else: for axis in self._axis_tags: if isinstance(input_data[axis], str): if self._project.hyperparameter[axis]["domain"] == "categorical": input_data[axis] = self._project.hyperparameter[axis]["data"].index(input_data[axis]) axis_loss = np.array([input_data[axis], input_data["loss"]]).reshape(1, -1) self._viz.scatter(axis_loss, win=self._axis_plots[axis], update='append') def show_statusreport(self, input_data): """ This function prints status report per iteration :param input_data: [dict] trail infos """ duration = input_data['refresh_time'] - input_data['book_time'] duration, time_format = time_formatter(duration.total_seconds()) report = "Iteration {}: {}{} -> {}\n".format(input_data["iterations"], duration, time_format, input_data["status"]) if self._status_report is None: self._status_report = self._viz.text(report) else: self._viz.text(report, win=self._status_report, append=True) def show_best(self, input_data): """ Shows best parameter set :param input_data: [dict] trail infos """ if self._best_win is None: self._best_loss = input_data["loss"] txt = "Best Parameter Set:
Loss: {}
" self._best_win = self._viz.text(txt) else: if input_data["loss"] < self._best_loss: self._best_loss = input_data["loss"] txt = "Best Parameter Set:
Loss: {}
" self._viz.text(txt, win=self._best_win, append=False) def update(self, input_data): """ This function calls all visdom displaying routines :param input_data: [dict] trail infos """ if self._enabled: self.show_statusreport(input_data) self.plot_losshistory(input_data) self.plot_hyperparameter(input_data) self.show_best(input_data) diff --git a/hyppopy/solvers/HyppopySolver.py b/hyppopy/solvers/HyppopySolver.py index c774ad4..b1dd916 100644 --- a/hyppopy/solvers/HyppopySolver.py +++ b/hyppopy/solvers/HyppopySolver.py @@ -1,445 +1,507 @@ # Hyppopy - A Hyper-Parameter Optimization Toolbox # # Copyright (c) German Cancer Research Center, # Division of Medical Image Computing. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE __all__ = ['HyppopySolver'] import abc import copy import types import datetime import numpy as np import pandas as pd from hyperopt import Trials from hyppopy.globals import * from hyppopy.VisdomViewer import VisdomViewer from hyppopy.HyppopyProject import HyppopyProject from hyppopy.BlackboxFunction import BlackboxFunction from hyppopy.FunctionSimulator import FunctionSimulator from hyppopy.globals import DEBUGLEVEL LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) class HyppopySolver(object): """ The HyppopySolver class is the base class for all solver addons. It defines virtual functions a child class has to implement to deal with the front-end communication, orchestrating the optimization process and ensuring a proper process information storing. The key idea is that the HyppopySolver class defines an interface to configure and run an object instance of itself independently from the concrete solver lib used to optimize in the background. To achieve this goal an addon developer needs to implement the abstract methods 'convert_searchspace', 'execute_solver' and 'loss_function_call'. These methods abstract the peculiarities of the solver libs to offer, on the user side, a simple and consistent parameter space configuration and optimization procedure. The method 'convert_searchspace' transforms the hyppopy parameter space description into the solver lib specific description. The method loss_function_call is used to handle solver lib specifics of calling the actual blackbox function and execute_solver is executed when the run method is invoked und takes care of calling the solver lib solving routine. The class HyppopySolver defines an interface to be implemented when writing a custom solver. Each solver derivative needs to implement the abstract methods: - convert_searchspace - execute_solver - loss_function_call - define_interface The dev-user interface consists of the methods: - _add_member - _add_hyperparameter_signature - _check_project The end-user interface consists of the methods: - run - get_results - print_best - print_timestats - start_viewer """ def __init__(self, project=None): self._idx = None # current iteration counter self._best = None # best parameter set self._trials = None # trials object, hyppopy uses the Trials object from hyperopt self._blackbox = None # blackbox function, eiter a function or a BlackboxFunction instance self._total_duration = None # keeps track of the solvers running time self._solver_overhead = None # stores the time overhead of the solver, means total time minus time in blackbox self._time_per_iteration = None # mean time per iterration self._accumulated_blackbox_time = None # summed time the solver was in the blackbox function self._visdom_viewer = None # visdom viewer instance self._child_members = {} # this dict keeps track of the settings the child solver defines self._hopt_signatures = {} # this dict keeps track of the hyperparameter signatures the child solver defines self.define_interface() # the child define interface function is called which defines settings and hyperparameter signatures if project is not None: self.project = project @abc.abstractmethod def convert_searchspace(self, hyperparameter): """ This function gets the unified hyppopy-like parameterspace description as input and, if necessary, should convert it into a solver lib specific format. The function is invoked when run is called and what it returns is passed as searchspace argument to the function execute_solver. :param hyperparameter: [dict] nested parameter description dict e.g. {'name': {'domain':'uniform', 'data':[0,1], 'type':'float'}, ...} :return: [object] converted hyperparameter space """ raise NotImplementedError('users must define convert_searchspace to use this class') @abc.abstractmethod def execute_solver(self, searchspace): """ This function is called immediatly after convert_searchspace and get the output of the latter as input. It's purpose is to call the solver libs main optimization function. :param searchspace: converted hyperparameter space """ raise NotImplementedError('users must define execute_solver to use this class') @abc.abstractmethod def loss_function_call(self, params): """ This function is called within the function loss_function and encapsulates the actual blackbox function call in each iteration. The function loss_function takes care of the iteration driving and reporting, but each solver lib might need some special treatment between the parameter set selection and the calling of the actual blackbox function, e.g. parameter converting. :param params: [dict] hyperparameter space sample e.g. {'p1': 0.123, 'p2': 3.87, ...} :return: [float] loss """ raise NotImplementedError('users must define loss_function_call to use this class') @abc.abstractmethod def define_interface(self): """ This function is called when HyppopySolver.__init__ function finished. Child classes need to define their individual parameter here by calling the _add_member function for each class member variable need to be defined. Using _add_hyperparameter_signature the structure of a hyperparameter the solver expects must be defined. Both, members and hyperparameter signatures are later get checked, before executing the solver, ensuring settings passed fullfill solver needs. """ raise NotImplementedError('users must define define_interface to use this class') def _add_member(self, name, dtype, value=None, default=None): """ When designing your child solver class you need to implement the define_interface abstract method where you can call _add_member to define custom solver options that are automatically converted to class attributes. :param name: [str] option name :param dtype: [type] option data type :param value: [object] option value :param default: [object] option default value """ assert isinstance(name, str), "precondition violation, name needs to be of type str, got {}".format(type(name)) if value is not None: assert isinstance(value, dtype), "precondition violation, value does not match dtype condition!" if default is not None: assert isinstance(default, dtype), "precondition violation, default does not match dtype condition!" setattr(self, name, value) self._child_members[name] = {"type": dtype, "value": value, "default": default} def _add_hyperparameter_signature(self, name, dtype, options=None): """ When designing your child solver class you need to implement the define_interface abstract method where you can call _add_hyperparameter_signature to define a hyperparamter signature which is automatically checked for consistency while solver execution. :param name: [str] hyperparameter name :param dtype: [type] hyperparameter data type :param options: [list] list of possible values the hp can be set, if None no option check is done """ assert isinstance(name, str), "precondition violation, name needs to be of type str, got {}".format(type(name)) self._hopt_signatures[name] = {"type": dtype, "options": options} def _check_project(self): """ The function checks the members and hyperparameter signatures read from the project instance to be consistent with the members and signatures defined in the child class via define_interface. """ assert isinstance(self.project, HyppopyProject), "Invalid project instance, either not set or setting failed!" # check hyperparameter signatures for name, param in self.project.hyperparameter.items(): for sig, settings in self._hopt_signatures.items(): if sig not in param.keys(): msg = "Missing hyperparameter signature {}!".format(sig) LOG.error(msg) raise LookupError(msg) else: if not isinstance(param[sig], settings["type"]): msg = "Hyperparameter signature type mismatch, expected type {} got {}!".format(settings["type"], param[sig]) LOG.error(msg) raise TypeError(msg) if settings["options"] is not None: if param[sig] not in settings["options"]: msg = "Wrong signature value, {} not found in signature options!".format(param[sig]) LOG.error(msg) raise LookupError(msg) # check child members for name in self._child_members.keys(): if name not in self.project.__dict__.keys(): msg = "missing settings field {}!".format(name) LOG.error(msg) raise LookupError(msg) self.__dict__[name] = self.project.settings[name] def __compute_time_statistics(self): """ Evaluates all timestatistic values available """ dts = [] for trial in self._trials.trials: if 'book_time' in trial.keys() and 'refresh_time' in trial.keys(): dt = trial['refresh_time'] - trial['book_time'] dts.append(dt.total_seconds()) self._time_per_iteration = np.mean(dts) * 1e3 self._accumulated_blackbox_time = np.sum(dts) * 1e3 tmp = self.total_duration - self._accumulated_blackbox_time self._solver_overhead = int(np.round(100.0 / (self.total_duration + 1e-12) * tmp)) def loss_function(self, **params): """ This function is called each iteration with a selected parameter set. The parameter set selection is driven by the solver lib itself. The purpose of this function is to take care of the iteration reporting and the calling of the callback_func is available. As a developer you might want to overwrite this function completely (e.g. HyperoptSolver) but then you need to take care for iteration reporting for yourself. The alternative is to only implement loss_function_call (e.g. OptunitySolver). :param params: [dict] hyperparameter space sample e.g. {'p1': 0.123, 'p2': 3.87, ...} :return: [float] loss """ self._idx += 1 vals = {} idx = {} for key, value in params.items(): vals[key] = [value] idx[key] = [self._idx] trial = {'tid': self._idx, 'result': {'loss': None, 'status': 'ok'}, 'misc': { 'tid': self._idx, 'idxs': idx, 'vals': vals }, 'book_time': datetime.datetime.now(), 'refresh_time': None } try: loss = self.loss_function_call(params) trial['result']['loss'] = loss trial['result']['status'] = 'ok' if loss == np.nan: trial['result']['status'] = 'failed' except Exception as e: LOG.error("computing loss failed due to:\n {}".format(e)) loss = np.nan trial['result']['loss'] = np.nan trial['result']['status'] = 'failed' trial['refresh_time'] = datetime.datetime.now() self._trials.trials.append(trial) cbd = copy.deepcopy(params) cbd['iterations'] = self._idx cbd['loss'] = loss cbd['status'] = trial['result']['status'] cbd['book_time'] = trial['book_time'] cbd['refresh_time'] = trial['refresh_time'] if isinstance(self.blackbox, BlackboxFunction) and self.blackbox.callback_func is not None: self.blackbox.callback_func(**cbd) if self._visdom_viewer is not None: self._visdom_viewer.update(cbd) return loss def run(self, print_stats=True): """ This function starts the optimization process. :param print_stats: [bool] en- or disable console output """ self._idx = 0 self.trials = Trials() start_time = datetime.datetime.now() try: search_space = self.convert_searchspace(self.project.hyperparameter) except Exception as e: msg = "Failed to convert searchspace, error: {}".format(e) LOG.error(msg) raise AssertionError(msg) try: self.execute_solver(search_space) except Exception as e: msg = "Failed to execute solver, error: {}".format(e) LOG.error(msg) raise AssertionError(msg) end_time = datetime.datetime.now() dt = end_time - start_time days = divmod(dt.total_seconds(), 86400) hours = divmod(days[1], 3600) minutes = divmod(hours[1], 60) seconds = divmod(minutes[1], 1) milliseconds = divmod(seconds[1], 0.001) self._total_duration = [int(days[0]), int(hours[0]), int(minutes[0]), int(seconds[0]), int(milliseconds[0])] if print_stats: self.print_best() self.print_timestats() def get_results(self): """ This function returns a complete optimization history as pandas DataFrame and a dict with the optimal parameter set. :return: [DataFrame], [dict] history and optimal parameter set """ assert isinstance(self.trials, Trials), "precondition violation, wrong trials type! Maybe solver was not yet executed?" results = {'duration': [], 'losses': [], 'status': []} pset = self.trials.trials[0]['misc']['vals'] for p in pset.keys(): results[p] = [] for n, trial in enumerate(self.trials.trials): t1 = trial['book_time'] t2 = trial['refresh_time'] results['duration'].append((t2 - t1).microseconds / 1000.0) results['losses'].append(trial['result']['loss']) results['status'].append(trial['result']['status'] == 'ok') losses = np.array(results['losses']) results['losses'] = list(losses) pset = trial['misc']['vals'] for p in pset.items(): results[p[0]].append(p[1][0]) return pd.DataFrame.from_dict(results), self.best def print_best(self): """ Optimization result console output printing. """ print("\n") print("#" * 40) print("### Best Parameter Choice ###") print("#" * 40) for name, value in self.best.items(): print(" - {}\t:\t{}".format(name, value)) print("\n - number of iterations\t:\t{}".format(self.trials.trials[-1]['tid']+1)) print(" - total time\t:\t{}d:{}h:{}m:{}s:{}ms".format(self._total_duration[0], self._total_duration[1], self._total_duration[2], self._total_duration[3], self._total_duration[4])) print("#" * 40) def print_timestats(self): """ Time statistic console output printing. """ print("\n") print("#" * 40) print("### Timing Statistics ###") print("#" * 40) print(" - per iteration: {}ms".format(int(self.time_per_iteration*1e4)/10000)) print(" - total time: {}d:{}h:{}m:{}s:{}ms".format(self._total_duration[0], self._total_duration[1], self._total_duration[2], self._total_duration[3], self._total_duration[4])) print("#" * 40) print(" - solver overhead: {}%".format(self.solver_overhead)) def start_viewer(self, port=8097, server="http://localhost"): """ Starts the visdom viewer. :param port: [int] port number, default: 8097 :param server: [str] server name, default: http://localhost """ try: self._visdom_viewer = VisdomViewer(self._project, port, server) except Exception as e: import warnings warnings.warn("Failed starting VisdomViewer. Is the server running? If not start it via $visdom") LOG.error("Failed starting VisdomViewer: {}".format(e)) self._visdom_viewer = None @property def project(self): + """ + HyppopyProject instance + + :return: [HyppopyProject] project instance + """ return self._project @project.setter def project(self, value): + """ + Set HyppopyProject instance + + :param value: [HyppopyProject] project instance + """ if isinstance(value, dict): self._project = HyppopyProject(value) elif isinstance(value, HyppopyProject): self._project = value else: msg = "Input error, project_manager of type: {} not allowed!".format(type(value)) LOG.error(msg) raise TypeError(msg) self._check_project() @property def blackbox(self): + """ + Get the BlackboxFunction object. + + :return: [object] BlackboxFunction instance or function + """ return self._blackbox @blackbox.setter def blackbox(self, value): + """ + Set the BlackboxFunction wrapper class encapsulating the loss function or a function accepting a hyperparameter set + and returning a float. + + :return: [object] pointer to blackbox_func + """ if isinstance(value, types.FunctionType) or isinstance(value, BlackboxFunction) or isinstance(value, FunctionSimulator): self._blackbox = value else: self._blackbox = None msg = "Input error, blackbox of type: {} not allowed!".format(type(value)) LOG.error(msg) raise TypeError(msg) @property def best(self): + """ + Returns best parameter set. + + :return: [dict] best parameter set + """ return self._best @best.setter def best(self, value): + """ + Set the best parameter set. + + :param value: [dict] best parameter set + + """ if not isinstance(value, dict): msg = "Input error, best of type: {} not allowed!".format(type(value)) LOG.error(msg) raise TypeError(msg) self._best = value @property def trials(self): + """ + Get the Trials instance. + + :return: [object] Trials instance + """ return self._trials @trials.setter def trials(self, value): + """ + Set the Trials object. + + :param value: [object] Trials instance + """ self._trials = value @property def total_duration(self): + """ + Get total computation duration. + + :return: [float] total computation time + """ return (self._total_duration[0]*86400 + self._total_duration[1] * 3600 + self._total_duration[2] * 60 + self._total_duration[3]) * 1000 + self._total_duration[4] @property def solver_overhead(self): + """ + Get the solver overhead, this is the total time minus the duration of the blackbox function calls. + + :return: [float] solver overhead duration + """ if self._solver_overhead is None: self.__compute_time_statistics() return self._solver_overhead @property def time_per_iteration(self): + """ + Get the mean duration per iteration. + + :return: [float] time per iteration + """ if self._time_per_iteration is None: self.__compute_time_statistics() return self._time_per_iteration @property def accumulated_blackbox_time(self): + """ + Get the summed blackbox function computation time. + + :return: [float] blackbox function computation time + """ if self._accumulated_blackbox_time is None: self.__compute_time_statistics() return self._accumulated_blackbox_time diff --git a/setup.py b/setup.py index 98abd06..f5b6e9b 100644 --- a/setup.py +++ b/setup.py @@ -1,54 +1,54 @@ import os from setuptools import setup, find_packages with open('README.md') as f: readme = f.read() with open('LICENSE') as f: license = f.read() -VERSION = "0.5.0.1" +VERSION = "0.5.0.2" ROOT = os.path.dirname(os.path.realpath(__file__)) new_init = [] with open(os.path.join(ROOT, *("hyppopy", "__init__.py")), "r") as infile: for line in infile: new_init.append(line) for n in range(len(new_init)): if new_init[n].startswith("__version__"): split = line.split("=") new_init[n] = "__version__ = '" + VERSION + "'\n" with open(os.path.join(ROOT, *("hyppopy", "__init__.py")), "w") as outfile: outfile.writelines(new_init) setup( name='hyppopy', version=VERSION, description='Hyper-Parameter Optimization Toolbox for Blackboxfunction Optimization', long_description=readme, # if you want, put your own name here # (this would likely result in people sending you emails) author='Sven Wanner', author_email='s.wanner@dkfz.de', url='', license=license, packages=find_packages(exclude=('tests', 'doc')), # the requirements to install this project. # Since this one is so simple this is empty. install_requires=[ 'bayesian-optimization>=1.0.1', 'hyperopt>=0.1.2', 'matplotlib>=3.0.3', 'numpy>=1.16.2', 'optuna>=0.9.0', 'Optunity>=1.1.1', 'pandas>=0.24.2', 'pytest>=4.3.1', 'scikit-learn>=0.20.3', 'scipy>=1.2.1', 'visdom>=0.1.8.8' ], )