diff --git a/hyppopy/plugins/optunity_solver_plugin.py b/hyppopy/plugins/optunity_solver_plugin.py index 6416920..895d3a5 100644 --- a/hyppopy/plugins/optunity_solver_plugin.py +++ b/hyppopy/plugins/optunity_solver_plugin.py @@ -1,71 +1,64 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical and Biological Informatics. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE.txt or http://www.mitk.org for details. # # Author: Sven Wanner (s.wanner@dkfz.de) import os import logging from hyppopy.globals import DEBUGLEVEL LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) from pprint import pformat import optunity from yapsy.IPlugin import IPlugin from hyppopy.projectmanager import ProjectManager from hyppopy.solverpluginbase import SolverPluginBase class optunity_Solver(SolverPluginBase, IPlugin): solver_info = None trials = None best = None status = None def __init__(self): SolverPluginBase.__init__(self) LOG.debug("initialized") def loss_function(self, **params): try: loss = self.loss(self.data, params) self.status.append('ok') return loss except Exception as e: LOG.error("computing loss failed due to:\n {}".format(e)) self.status.append('fail') return 1e9 def execute_solver(self, parameter): LOG.debug("execute_solver using solution space:\n\n\t{}\n".format(pformat(parameter))) self.status = [] try: self.best, self.trials, self.solver_info = optunity.minimize_structured(f=self.loss_function, num_evals=ProjectManager.max_iterations, search_space=parameter) except Exception as e: LOG.error("internal error in optunity.minimize_structured occured. {}".format(e)) raise BrokenPipeError("internal error in optunity.minimize_structured occured. {}".format(e)) def convert_results(self): - solution = dict([(k, v) for k, v in self.best.items() if v is not None]) - - txt = "" - txt += 'Solution Optunity Plugin\n========\n' - txt += "\n".join(map(lambda x: "%s \t %s" % (x[0], str(x[1])), solution.items())) - txt += "\nSolver used: {}".format(self.solver_info['solver_name']) - txt += "\nOptimum: {}".format(self.trials.optimum) - txt += "\nIterations used: {}".format(self.trials.stats['num_evals']) - txt += "\nDuration: {} s\n".format(self.trials.stats['time']) - return txt + results = self.trials.call_log['args'] + results['losses'] = self.trials.call_log['values'] + return results, self.best diff --git a/hyppopy/tests/test_solver_factory.py b/hyppopy/tests/test_solver_factory.py index 00c1b1c..b605055 100644 --- a/hyppopy/tests/test_solver_factory.py +++ b/hyppopy/tests/test_solver_factory.py @@ -1,105 +1,99 @@ # -*- coding: utf-8 -*- # # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical and Biological Informatics. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE.txt or http://www.mitk.org for details. # # Author: Sven Wanner (s.wanner@dkfz.de) import os import unittest from sklearn.svm import SVC from sklearn import datasets from sklearn.model_selection import cross_val_score from sklearn.model_selection import train_test_split from hyppopy.solverfactory import SolverFactory from hyppopy.projectmanager import ProjectManager from hyppopy.globals import TESTDATA_DIR TESTPARAMFILE = os.path.join(TESTDATA_DIR, 'iris_svc_parameter') from hyppopy.deepdict import DeepDict class SolverFactoryTestSuite(unittest.TestCase): def setUp(self): iris = datasets.load_iris() X, X_test, y, y_test = train_test_split(iris.data, iris.target, test_size=0.1, random_state=42) self.my_IRIS_dta = [X, y] def test_solver_loading(self): names = SolverFactory.list_solver() self.assertTrue("hyperopt" in names) self.assertTrue("optunity" in names) def test_iris_solver_execution(self): def my_SVC_loss_func(data, params): clf = SVC(**params) return -cross_val_score(clf, data[0], data[1], cv=3).mean() ProjectManager.read_config(TESTPARAMFILE + '.xml') solver = SolverFactory.get_solver('optunity') solver.set_data(self.my_IRIS_dta) solver.set_loss_function(my_SVC_loss_func) solver.run() - solver.get_results() ProjectManager.read_config(TESTPARAMFILE + '.json') solver = SolverFactory.get_solver('hyperopt') solver.set_data(self.my_IRIS_dta) solver.set_loss_function(my_SVC_loss_func) solver.run() - solver.get_results() def test_create_solver_from_settings_directly(self): def my_SVC_loss_func(data, params): clf = SVC(**params) return -cross_val_score(clf, data[0], data[1], cv=3).mean() solver = SolverFactory.from_settings(TESTPARAMFILE + '.xml') self.assertEqual(solver.name, "optunity") solver.set_data(self.my_IRIS_dta) solver.set_loss_function(my_SVC_loss_func) solver.run() - solver.get_results() solver = SolverFactory.from_settings(TESTPARAMFILE + '.json') self.assertEqual(solver.name, "hyperopt") solver.set_data(self.my_IRIS_dta) solver.set_loss_function(my_SVC_loss_func) solver.run() - solver.get_results() dd = DeepDict(TESTPARAMFILE + '.json') solver = SolverFactory.from_settings(dd) self.assertEqual(solver.name, "hyperopt") solver.set_data(self.my_IRIS_dta) solver.set_loss_function(my_SVC_loss_func) solver.run() - solver.get_results() solver = SolverFactory.from_settings(dd.data) self.assertEqual(solver.name, "hyperopt") solver.set_data(self.my_IRIS_dta) solver.set_loss_function(my_SVC_loss_func) solver.run() - solver.get_results() if __name__ == '__main__': unittest.main() diff --git a/hyppopy/tests/test_workflows.py b/hyppopy/tests/test_workflows.py index 6b2c848..508e946 100644 --- a/hyppopy/tests/test_workflows.py +++ b/hyppopy/tests/test_workflows.py @@ -1,96 +1,82 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical and Biological Informatics. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE.txt or http://www.mitk.org for details. # # Author: Sven Wanner (s.wanner@dkfz.de) import os import unittest from hyppopy.globals import TESTDATA_DIR IRIS_DATA = os.path.join(TESTDATA_DIR, 'Iris') TITANIC_DATA = os.path.join(TESTDATA_DIR, 'Titanic') from hyppopy.projectmanager import ProjectManager from hyppopy.workflows.svc_usecase.svc_usecase import svc_usecase from hyppopy.workflows.randomforest_usecase.randomforest_usecase import randomforest_usecase class WorkflowTestSuite(unittest.TestCase): def setUp(self): self.results = [] def test_workflow_svc_on_iris_from_xml(self): ProjectManager.read_config(os.path.join(IRIS_DATA, 'svc_config.xml')) uc = svc_usecase() - uc.run() - self.results.append(uc.get_results()) - self.assertTrue(uc.get_results().find("Solution") != -1) + uc.run(False) + res, best = uc.get_results() + self.assertEqual(len(best.keys()), 4) def test_workflow_rf_on_iris_from_xml(self): ProjectManager.read_config(os.path.join(IRIS_DATA, 'rf_config.xml')) uc = svc_usecase() - uc.run() - self.results.append(uc.get_results()) - self.assertTrue(uc.get_results().find("Solution") != -1) + uc.run(False) + res, best = uc.get_results() + self.assertEqual(len(best.keys()), 6) def test_workflow_svc_on_iris_from_json(self): ProjectManager.read_config(os.path.join(IRIS_DATA, 'svc_config.json')) uc = svc_usecase() - uc.run() - self.results.append(uc.get_results()) - self.assertTrue(uc.get_results().find("Solution") != -1) + uc.run(False) + res, best = uc.get_results() + self.assertEqual(len(best.keys()), 4) def test_workflow_rf_on_iris_from_json(self): ProjectManager.read_config(os.path.join(IRIS_DATA, 'rf_config.json')) uc = randomforest_usecase() - uc.run() - self.results.append(uc.get_results()) - self.assertTrue(uc.get_results().find("Solution") != -1) - - # def test_workflow_svc_on_titanic_from_xml(self): - # ProjectManager.read_config(os.path.join(TITANIC_DATA, 'svc_config.xml')) - # uc = svc_usecase() - # uc.run() - # self.results.append(uc.get_results()) - # self.assertTrue(uc.get_results().find("Solution") != -1) + uc.run(False) + res, best = uc.get_results() + self.assertEqual(len(best.keys()), 6) def test_workflow_rf_on_titanic_from_xml(self): ProjectManager.read_config(os.path.join(TITANIC_DATA, 'rf_config.xml')) uc = randomforest_usecase() - uc.run() - self.results.append(uc.get_results()) - self.assertTrue(uc.get_results().find("Solution") != -1) - - # def test_workflow_svc_on_titanic_from_json(self): - # ProjectManager.read_config(os.path.join(TITANIC_DATA, 'svc_config.json')) - # uc = svc_usecase() - # uc.run() - # self.results.append(uc.get_results()) - # self.assertTrue(uc.get_results().find("Solution") != -1) + uc.run(False) + res, best = uc.get_results() + self.assertEqual(len(best.keys()), 3) def test_workflow_rf_on_titanic_from_json(self): ProjectManager.read_config(os.path.join(TITANIC_DATA, 'rf_config.json')) uc = randomforest_usecase() - uc.run() - self.results.append(uc.get_results()) - self.assertTrue(uc.get_results().find("Solution") != -1) + uc.run(False) + res, best = uc.get_results() + self.assertEqual(len(best.keys()), 3) def tearDown(self): print("") for r in self.results: print(r) if __name__ == '__main__': unittest.main()