diff --git a/bin/hyppopy_exe.py b/bin/hyppopy_exe.py index 6dc44e8..36f12f1 100644 --- a/bin/hyppopy_exe.py +++ b/bin/hyppopy_exe.py @@ -1,76 +1,90 @@ #!/usr/bin/env python # # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical and Biological Informatics. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE.txt or http://www.mitk.org for details. # # Author: Sven Wanner (s.wanner@dkfz.de) import os import sys ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) sys.path.append(ROOT) from hyppopy.projectmanager import ProjectManager from hyppopy.workflows.unet_usecase.unet_usecase import unet_usecase from hyppopy.workflows.svc_usecase.svc_usecase import svc_usecase from hyppopy.workflows.randomforest_usecase.randomforest_usecase import randomforest_usecase from hyppopy.workflows.imageregistration_usecase.imageregistration_usecase import imageregistration_usecase import os import sys +import time import argparse def print_warning(msg): print("\n!!!!! WARNING !!!!!") print(msg) sys.exit() def args_check(args): if not args.workflow: print_warning("No workflow specified, check --help") if not args.config: print_warning("Missing config parameter, check --help") if not os.path.isfile(args.config): print_warning(f"Couldn't find configfile ({args.config}), please check your input --config") if __name__ == "__main__": parser = argparse.ArgumentParser(description='UNet Hyppopy UseCase Example Optimization.') parser.add_argument('-w', '--workflow', type=str, help='workflow to be executed') + parser.add_argument('-o', '--output', type=str, default=None) parser.add_argument('-c', '--config', type=str, help='config filename, .xml or .json formats are supported.' 'pass a full path filename or the filename only if the' 'configfile is in the data folder') args = parser.parse_args() args_check(args) ProjectManager.read_config(args.config) + if args.output is not None: + ProjectManager.output_dir = args.output + if args.workflow == "svc_usecase": uc = svc_usecase() elif args.workflow == "randomforest_usecase": uc = randomforest_usecase() elif args.workflow == "unet_usecase": uc = unet_usecase() elif args.workflow == "imageregistration_usecase": uc = imageregistration_usecase() else: print("No workflow called {} found!".format(args.workflow)) sys.exit() + print("\nStart optimization...") + start = time.process_time() uc.run() - print(uc.get_results()) + end = time.process_time() + + print("Finished optimization!\n") + print("Total Time: {}s\n".format(end-start)) + res, best = uc.get_results() + print("---- Optimal Parameter -----\n") + for p in best.items(): + print(" - {}\t:\t{}".format(p[0], p[1])) diff --git a/hyppopy/plugins/hyperopt_solver_plugin.py b/hyppopy/plugins/hyperopt_solver_plugin.py index a8854ec..587bf07 100644 --- a/hyppopy/plugins/hyperopt_solver_plugin.py +++ b/hyppopy/plugins/hyperopt_solver_plugin.py @@ -1,69 +1,80 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical and Biological Informatics. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE.txt or http://www.mitk.org for details. # # Author: Sven Wanner (s.wanner@dkfz.de) import os import logging from hyppopy.globals import DEBUGLEVEL LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) from pprint import pformat from hyperopt import fmin, tpe, hp, STATUS_OK, STATUS_FAIL, Trials from yapsy.IPlugin import IPlugin from hyppopy.projectmanager import ProjectManager from hyppopy.solverpluginbase import SolverPluginBase class hyperopt_Solver(SolverPluginBase, IPlugin): trials = None best = None def __init__(self): SolverPluginBase.__init__(self) LOG.debug("initialized") def loss_function(self, params): try: loss = self.loss(self.data, params) status = STATUS_OK except Exception as e: LOG.error("execution of self.loss(self.data, params) failed due to:\n {}".format(e)) status = STATUS_FAIL return {'loss': loss, 'status': status} def execute_solver(self, parameter): LOG.debug("execute_solver using solution space:\n\n\t{}\n".format(pformat(parameter))) self.trials = Trials() try: self.best = fmin(fn=self.loss_function, space=parameter, algo=tpe.suggest, max_evals=ProjectManager.max_iterations, trials=self.trials) except Exception as e: msg = "internal error in hyperopt.fmin occured. {}".format(e) LOG.error(msg) raise BrokenPipeError(msg) def convert_results(self): - txt = "" - solution = dict([(k, v) for k, v in self.best.items() if v is not None]) - txt += 'Solution Hyperopt Plugin\n========\n' - txt += "\n".join(map(lambda x: "%s \t %s" % (x[0], str(x[1])), solution.items())) - txt += "\n" - return txt + # currently converting results in a way that this function returns a dict + # keeping all useful parameter as key/list item. This will be automatically + # converted to a pandas dataframe in the solver class + results = {'timing ms': [], 'losses': []} + pset = self.trials.trials[0]['misc']['vals'] + for p in pset.keys(): + results[p] = [] + + for n, trial in enumerate(self.trials.trials): + t1 = trial['book_time'] + t2 = trial['refresh_time'] + results['timing ms'].append((t2 - t1).microseconds/1000.0) + results['losses'].append(trial['result']['loss']) + pset = trial['misc']['vals'] + for p in pset.items(): + results[p[0]].append(p[1][0]) + return results, self.best diff --git a/hyppopy/solver.py b/hyppopy/solver.py index 08e7d83..2ceb972 100644 --- a/hyppopy/solver.py +++ b/hyppopy/solver.py @@ -1,86 +1,118 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical and Biological Informatics. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE.txt or http://www.mitk.org for details. # # Author: Sven Wanner (s.wanner@dkfz.de) from hyppopy.projectmanager import ProjectManager import os +import datetime import logging +import pandas as pd from hyppopy.globals import DEBUGLEVEL LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) class Solver(object): _name = None _solver_plugin = None _settings_plugin = None def __init__(self): pass def set_data(self, data): self.solver.set_data(data) def set_hyperparameters(self, params): self.settings.set_hyperparameter(params) def set_loss_function(self, loss_func): self.solver.set_loss_function(loss_func) def run(self): if not ProjectManager.is_ready(): LOG.error("No config data found to initialize PluginSetting object") raise IOError("No config data found to initialize PluginSetting object") hyps = ProjectManager.get_hyperparameter() self.settings.set_hyperparameter(hyps) self.solver.settings = self.settings self.solver.run() + def save_results(self, savedir=None, savename=None): + df, best = self.get_results() + dir = None + if savename is None: + savename = "hypopy" + if savedir is None: + if 'output_dir' in ProjectManager.__dict__.keys(): + if not os.path.isdir(ProjectManager.output_dir): + os.mkdir(ProjectManager.output_dir) + dir = ProjectManager.output_dir + else: + print("WARNING: No solver option output_dir found, cannot save results!") + LOG.warning("WARNING: No solver option output_dir found, cannot save results!") + else: + if not os.path.isdir(savedir): + os.mkdir(savedir) + dir = savedir + + tstr = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + name = savename + "_all_" + tstr + ".csv" + fname = os.path.join(dir, name) + df.to_csv(fname) + name = savename + "_best_" + tstr + ".txt" + fname = os.path.join(dir, name) + with open(fname, "w") as text_file: + for item in best.items(): + text_file.write("{}\t:\t{}\n".format(item[0], item[1])) + def get_results(self): - return self.solver.get_results() + results, best = self.solver.get_results() + df = pd.DataFrame.from_dict(results) + return df, best @property def is_ready(self): return self.solver is not None and self.settings is not None @property def solver(self): return self._solver_plugin @solver.setter def solver(self, value): self._solver_plugin = value @property def settings(self): return self._settings_plugin @settings.setter def settings(self, value): self._settings_plugin = value @property def name(self): return self._name @name.setter def name(self, value): if not isinstance(value, str): msg = "Invalid input, str type expected for value, got {} instead".format(type(value)) LOG.error(msg) raise IOError(msg) self._name = value diff --git a/hyppopy/solverfactory.py b/hyppopy/solverfactory.py index 17ff3e8..d5e3246 100644 --- a/hyppopy/solverfactory.py +++ b/hyppopy/solverfactory.py @@ -1,158 +1,158 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical and Biological Informatics. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE.txt or http://www.mitk.org for details. # # Author: Sven Wanner (s.wanner@dkfz.de) from yapsy.PluginManager import PluginManager from hyppopy.projectmanager import ProjectManager from hyppopy.globals import PLUGIN_DEFAULT_DIR from hyppopy.deepdict import DeepDict from hyppopy.solver import Solver from hyppopy.singleton import * import os import logging from hyppopy.globals import DEBUGLEVEL LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) @singleton_object class SolverFactory(metaclass=Singleton): """ This class is responsible for grabbing all plugins from the plugin folder arranging them into a Solver class instances. These Solver class instances can be requested from the factory via the get_solver method. The SolverFactory class is a Singleton class, so try not to instantiate it using SolverFactory(), the consequences will be horrific. Instead use is like a class having static functions only, SolverFactory.method(). """ _plugin_dirs = [] _plugins = {} def __init__(self): - print("Solverfactory: I'am alive!") self.reset() self.load_plugins() LOG.debug("Solverfactory initialized") def load_plugins(self): """ Load plugin modules from plugin paths """ LOG.debug("load_plugins()") manager = PluginManager() LOG.debug("setPluginPlaces(" + " ".join(map(str, self._plugin_dirs))) manager.setPluginPlaces(self._plugin_dirs) manager.collectPlugins() for plugin in manager.getAllPlugins(): name_elements = plugin.plugin_object.__class__.__name__.split("_") LOG.debug("found plugin " + " ".join(map(str, name_elements))) + print("Solverfactory: found plugins " + " ".join(map(str, name_elements))) if len(name_elements) != 2 or ("Solver" not in name_elements and "Settings" not in name_elements): msg = "invalid plugin class naming for class {}, the convention is libname_Solver or libname_Settings.".format(plugin.plugin_object.__class__.__name__) LOG.error(msg) raise NameError(msg) if name_elements[0] not in self._plugins.keys(): self._plugins[name_elements[0]] = Solver() self._plugins[name_elements[0]].name = name_elements[0] if name_elements[1] == "Solver": try: obj = plugin.plugin_object.__class__() obj.name = name_elements[0] self._plugins[name_elements[0]].solver = obj LOG.info("plugin: {} Solver loaded".format(name_elements[0])) except Exception as e: msg = "failed to instanciate class {}".format(plugin.plugin_object.__class__.__name__) LOG.error(msg) raise ImportError(msg) elif name_elements[1] == "Settings": try: obj = plugin.plugin_object.__class__() obj.name = name_elements[0] self._plugins[name_elements[0]].settings = obj LOG.info("plugin: {} ParameterSpace loaded".format(name_elements[0])) except Exception as e: msg = "failed to instanciate class {}".format(plugin.plugin_object.__class__.__name__) LOG.error(msg) raise ImportError(msg) else: msg = "failed loading plugin {}, please check if naming conventions are kept!".format(name_elements[0]) LOG.error(msg) raise IOError(msg) if len(self._plugins) == 0: msg = "no plugins found, please check your plugin folder names or your plugin scripts for errors!" LOG.error(msg) raise IOError(msg) def reset(self): """ Reset solver factory """ LOG.debug("reset()") self._plugins = {} self._plugin_dirs = [] self.add_plugin_dir(os.path.abspath(PLUGIN_DEFAULT_DIR)) def add_plugin_dir(self, dir): """ Add plugin directory """ LOG.debug("add_plugin_dir({})".format(dir)) self._plugin_dirs.append(dir) def list_solver(self): """ list all solvers available :return: [list(str)] """ return list(self._plugins.keys()) def from_settings(self, settings): if isinstance(settings, str): if not os.path.isfile(settings): LOG.error("input error, file {} not found!".format(settings)) if not ProjectManager.read_config(settings): LOG.error("failed to read config in ProjectManager!") return None else: if not ProjectManager.set_config(settings): LOG.error("failed to set config in ProjectManager!") return None if not ProjectManager.is_ready(): LOG.error("failed to set config in ProjectManager!") return None try: solver = self.get_solver(ProjectManager.use_plugin) except Exception as e: msg = "failed to create solver, reason {}".format(e) LOG.error(msg) return None return solver def get_solver(self, name): """ returns a solver by name tag :param name: [str] solver name :return: [Solver] instance """ if not isinstance(name, str): msg = "Invalid input, str type expected for name, got {} instead".format(type(name)) LOG.error(msg) raise IOError(msg) if name not in self.list_solver(): msg = "failed solver request, a solver called {} is not available, check for typo or if your plugin failed while loading!".format(name) LOG.error(msg) raise LookupError(msg) LOG.debug("get_solver({})".format(name)) return self._plugins[name] diff --git a/hyppopy/solverpluginbase.py b/hyppopy/solverpluginbase.py index 10944a3..795f5d3 100644 --- a/hyppopy/solverpluginbase.py +++ b/hyppopy/solverpluginbase.py @@ -1,84 +1,90 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical and Biological Informatics. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE.txt or http://www.mitk.org for details. # # Author: Sven Wanner (s.wanner@dkfz.de) import abc import os +import time import logging from hyppopy.globals import DEBUGLEVEL from hyppopy.settingspluginbase import SettingsPluginBase LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) class SolverPluginBase(object): data = None loss = None _settings = None _name = None + _timer = [] def __init__(self): pass @abc.abstractmethod def loss_function(self, params): raise NotImplementedError('users must define loss_func to use this base class') @abc.abstractmethod def execute_solver(self): raise NotImplementedError('users must define execute_solver to use this base class') @abc.abstractmethod def convert_results(self): raise NotImplementedError('users must define convert_results to use this base class') def set_data(self, data): self.data = data + self._timer = [] def set_loss_function(self, func): self.loss = func def get_results(self): return self.convert_results() def run(self): + start = time.time() self.execute_solver(self.settings.get_hyperparameter()) + end = time.time() + self._timer.append(end - start) @property def name(self): return self._name @name.setter def name(self, value): if not isinstance(value, str): msg = "Invalid input, str type expected for value, got {} instead".format(type(value)) LOG.error(msg) raise IOError(msg) self._name = value @property def settings(self): return self._settings @settings.setter def settings(self, value): if not isinstance(value, SettingsPluginBase): msg = "Invalid input, SettingsPluginBase type expected for value, got {} instead".format(type(value)) LOG.error(msg) raise IOError(msg) self._settings = value diff --git a/hyppopy/tests/data/Iris/rf_config.json b/hyppopy/tests/data/Iris/rf_config.json index 443e510..59ecd73 100644 --- a/hyppopy/tests/data/Iris/rf_config.json +++ b/hyppopy/tests/data/Iris/rf_config.json @@ -1,43 +1,44 @@ {"hyperparameter": { "n_estimators": { "domain": "uniform", "data": "[3,500]", "type": "int" }, "criterion": { "domain": "categorical", "data": "[gini,entropy]", "type": "str" }, "max_depth": { "domain": "uniform", "data": "[3, 50]", "type": "int" }, "min_samples_split": { "domain": "uniform", "data": "[0.0001,1]", "type": "float" }, "min_samples_leaf": { "domain": "uniform", "data": "[0.0001,0.5]", "type": "float" }, "max_features": { "domain": "categorical", "data": "[auto,sqrt,log2]", "type": "str" } }, "settings": { "solver": { "max_iterations": "3", - "use_plugin" : "optunity" + "use_plugin" : "optunity", + "output_dir": "D:/Projects/Python/hyppopy/hyppopy/tests/data/Iris" }, "custom": { "data_path": "D:/Projects/Python/hyppopy/hyppopy/tests/data/Iris", "data_name": "train_data.npy", "labels_name": "train_labels.npy" } }} \ No newline at end of file diff --git a/hyppopy/tests/data/Iris/rf_config.xml b/hyppopy/tests/data/Iris/rf_config.xml index b60530d..516eb43 100644 --- a/hyppopy/tests/data/Iris/rf_config.xml +++ b/hyppopy/tests/data/Iris/rf_config.xml @@ -1,45 +1,46 @@ uniform [3,200] int categorical [gini,entropy] str uniform [3, 50] int uniform [0.0001,1] float uniform [0.0001,0.5] float categorical [auto,sqrt,log2] str 3 optunity + D:/Projects/Python/hyppopy/hyppopy/tests/data/Iris D:/Projects/Python/hyppopy/hyppopy/tests/data/Iris train_data.npy train_labels.npy \ No newline at end of file diff --git a/hyppopy/tests/data/Iris/svc_config.json b/hyppopy/tests/data/Iris/svc_config.json index 59fb433..2ed1c8d 100644 --- a/hyppopy/tests/data/Iris/svc_config.json +++ b/hyppopy/tests/data/Iris/svc_config.json @@ -1,33 +1,34 @@ {"hyperparameter": { "C": { "domain": "uniform", "data": "[0,20]", "type": "float" }, "gamma": { "domain": "uniform", "data": "[0.0001,20.0]", "type": "float" }, "kernel": { "domain": "categorical", "data": "[linear, sigmoid, poly, rbf]", "type": "str" }, "decision_function_shape": { "domain": "categorical", "data": "[ovo,ovr]", "type": "str" } }, "settings": { "solver": { "max_iterations": "3", - "use_plugin" : "optunity" + "use_plugin" : "optunity", + "output_dir": "D:/Projects/Python/hyppopy/hyppopy/tests/data/Iris" }, "custom": { "data_path": "D:/Projects/Python/hyppopy/hyppopy/tests/data/Iris", "data_name": "train_data.npy", "labels_name": "train_labels.npy" } }} \ No newline at end of file diff --git a/hyppopy/tests/data/Iris/svc_config.xml b/hyppopy/tests/data/Iris/svc_config.xml index f8ab4e3..2495f60 100644 --- a/hyppopy/tests/data/Iris/svc_config.xml +++ b/hyppopy/tests/data/Iris/svc_config.xml @@ -1,35 +1,36 @@ uniform [0,20] float uniform [0.0001,20.0] float categorical [linear,sigmoid,poly,rbf] str categorical [ovo,ovr] str 3 hyperopt + D:/Projects/Python/hyppopy/hyppopy/tests/data/Iris D:/Projects/Python/hyppopy/hyppopy/tests/data/Iris train_data.npy train_labels.npy \ No newline at end of file diff --git a/hyppopy/tests/data/Titanic/rf_config.json b/hyppopy/tests/data/Titanic/rf_config.json index a637f35..af66f3a 100644 --- a/hyppopy/tests/data/Titanic/rf_config.json +++ b/hyppopy/tests/data/Titanic/rf_config.json @@ -1,28 +1,29 @@ {"hyperparameter": { "n_estimators": { "domain": "uniform", "data": "[3,500]", "type": "int" }, "criterion": { "domain": "categorical", "data": "[gini,entropy]", "type": "str" }, "max_depth": { "domain": "uniform", "data": "[3, 50]", "type": "int" } }, "settings": { "solver": { "max_iterations": "3", - "use_plugin" : "optunity" + "use_plugin" : "optunity", + "output_dir": "D:/Projects/Python/hyppopy/hyppopy/tests/data/Titanic" }, "custom": { "data_path": "D:/Projects/Python/hyppopy/hyppopy/tests/data/Titanic", "data_name": "train_cleaned.csv", "labels_name": "Survived" } }} \ No newline at end of file diff --git a/hyppopy/tests/data/Titanic/rf_config.xml b/hyppopy/tests/data/Titanic/rf_config.xml index fbfa828..a812834 100644 --- a/hyppopy/tests/data/Titanic/rf_config.xml +++ b/hyppopy/tests/data/Titanic/rf_config.xml @@ -1,30 +1,31 @@ uniform [3,200] int categorical [gini,entropy] str uniform [3, 50] int 3 optunity + D:/Projects/Python/hyppopy/hyppopy/tests/data/Titanic D:/Projects/Python/hyppopy/hyppopy/tests/data/Titanic train_cleaned.csv Survived \ No newline at end of file diff --git a/hyppopy/tests/data/Titanic/svc_config.json b/hyppopy/tests/data/Titanic/svc_config.json index 4066bb6..98eced5 100644 --- a/hyppopy/tests/data/Titanic/svc_config.json +++ b/hyppopy/tests/data/Titanic/svc_config.json @@ -1,33 +1,34 @@ {"hyperparameter": { "C": { "domain": "uniform", "data": "[0,20]", "type": "float" }, "gamma": { "domain": "uniform", "data": "[0.0001,20.0]", "type": "float" }, "kernel": { "domain": "categorical", "data": "[linear, sigmoid, poly, rbf]", "type": "str" }, "decision_function_shape": { "domain": "categorical", "data": "[ovo,ovr]", "type": "str" } }, "settings": { "solver": { "max_iterations": "3", - "use_plugin" : "hyperopt" + "use_plugin" : "hyperopt", + "output_dir": "D:/Projects/Python/hyppopy/hyppopy/tests/data/Titanic" }, "custom": { "data_path": "D:/Projects/Python/hyppopy/hyppopy/tests/data/Titanic", "data_name": "train_cleaned.csv", "labels_name": "Survived" } }} \ No newline at end of file diff --git a/hyppopy/tests/data/Titanic/svc_config.xml b/hyppopy/tests/data/Titanic/svc_config.xml index 1107c4a..60ed08e 100644 --- a/hyppopy/tests/data/Titanic/svc_config.xml +++ b/hyppopy/tests/data/Titanic/svc_config.xml @@ -1,35 +1,36 @@ uniform [0,20] float uniform [0.0001,20.0] float categorical [linear,sigmoid,poly,rbf] str categorical [ovo,ovr] str 3 optunity + D:/Projects/Python/hyppopy/hyppopy/tests/data/Titanic D:/Projects/Python/hyppopy/hyppopy/tests/data/Titanic train_cleaned.csv Survived \ No newline at end of file diff --git a/hyppopy/workflows/workflowbase.py b/hyppopy/workflows/workflowbase.py index f66b878..3b8a8a5 100644 --- a/hyppopy/workflows/workflowbase.py +++ b/hyppopy/workflows/workflowbase.py @@ -1,61 +1,63 @@ # -*- coding: utf-8 -*- # # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical and Biological Informatics. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE.txt or http://www.mitk.org for details. # # Author: Sven Wanner (s.wanner@dkfz.de) from hyppopy.deepdict import DeepDict from hyppopy.solverfactory import SolverFactory from hyppopy.projectmanager import ProjectManager from hyppopy.globals import SETTINGSCUSTOMPATH, SETTINGSSOLVERPATH import os import abc import logging from hyppopy.globals import DEBUGLEVEL LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) class WorkflowBase(object): def __init__(self): self._solver = SolverFactory.get_solver(ProjectManager.use_plugin) self.solver.set_hyperparameters(ProjectManager.get_hyperparameter()) - def run(self): + def run(self, save=True): self.setup() self.solver.set_loss_function(self.blackbox_function) self.solver.run() + if save: + self.solver.save_results() self.test() def get_results(self): return self.solver.get_results() @abc.abstractmethod def setup(self): raise NotImplementedError('the user has to implement this function') @abc.abstractmethod def blackbox_function(self): raise NotImplementedError('the user has to implement this function') @abc.abstractmethod def test(self): pass @property def solver(self): return self._solver