diff --git a/__main__.py b/__main__.py index a720673..552b78d 100644 --- a/__main__.py +++ b/__main__.py @@ -1,95 +1,95 @@ #!/usr/bin/env python # # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical and Biological Informatics. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE.txt or http://www.mitk.org for details. # # Author: Sven Wanner (s.wanner@dkfz.de) import os import sys ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) sys.path.append(ROOT) from hyppopy.projectmanager import ProjectManager from hyppopy.workflows.svc_usecase.svc_usecase import svc_usecase from hyppopy.workflows.knc_usecase.knc_usecase import knc_usecase from hyppopy.workflows.lda_usecase.lda_usecase import lda_usecase from hyppopy.workflows.unet_usecase.unet_usecase import unet_usecase from hyppopy.workflows.randomforest_usecase.randomforest_usecase import randomforest_usecase from hyppopy.workflows.imageregistration_usecase.imageregistration_usecase import imageregistration_usecase import os import sys import time import argparse def print_warning(msg): print("\n!!!!! WARNING !!!!!") print(msg) sys.exit() def args_check(args): if not args.workflow: print_warning("No workflow specified, check --help") if not args.config: print_warning("Missing config parameter, check --help") if not os.path.isfile(args.config): print_warning(f"Couldn't find configfile ({args.config}), please check your input --config") if __name__ == "__main__": parser = argparse.ArgumentParser(description='UNet Hyppopy UseCase Example Optimization.') parser.add_argument('-w', '--workflow', type=str, help='workflow to be executed') parser.add_argument('-o', '--output', type=str, default=None, help='output path to store result') parser.add_argument('-c', '--config', type=str, help='config filename, .xml or .json formats are supported.' 'pass a full path filename or the filename only if the' 'configfile is in the data folder') args = parser.parse_args() args_check(args) ProjectManager.read_config(args.config) if args.output is not None: - ProjectManager.output_dir = args.output + ProjectManager.register_member("output_dir", args.output) if args.workflow == "svc_usecase": uc = svc_usecase() elif args.workflow == "randomforest_usecase": uc = randomforest_usecase() elif args.workflow == "knc_usecase": uc = knc_usecase() elif args.workflow == "lda_usecase": uc = lda_usecase() elif args.workflow == "unet_usecase": uc = unet_usecase() elif args.workflow == "imageregistration_usecase": uc = imageregistration_usecase() else: print("No workflow called {} found!".format(args.workflow)) sys.exit() print("\nStart optimization...") start = time.process_time() uc.run(save=True) end = time.process_time() print("Finished optimization!\n") print("Total Time: {}s\n".format(end-start)) res, best = uc.get_results() print("---- Optimal Parameter -----\n") for p in best.items(): print(" - {}\t:\t{}".format(p[0], p[1])) diff --git a/hyppopy/projectmanager.py b/hyppopy/projectmanager.py index efae813..af987b9 100644 --- a/hyppopy/projectmanager.py +++ b/hyppopy/projectmanager.py @@ -1,147 +1,150 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical and Biological Informatics. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE.txt or http://www.mitk.org for details. # # Author: Sven Wanner (s.wanner@dkfz.de) from hyppopy.singleton import * from hyppopy.deepdict import DeepDict from hyppopy.globals import SETTINGSCUSTOMPATH, SETTINGSSOLVERPATH import os import logging import datetime from hyppopy.globals import DEBUGLEVEL LOG = logging.getLogger(os.path.basename(__file__)) LOG.setLevel(DEBUGLEVEL) @singleton_object class ProjectManager(metaclass=Singleton): def __init__(self): self.configfilename = None self.config = None self._extmembers = [] self._identifier = None def clear(self): self.configfilename = None self.config = None self.remove_externals() def is_ready(self): return self.config is not None def remove_externals(self): for added in self._extmembers: if added in self.__dict__.keys(): del self.__dict__[added] self._extmembers = [] def get_hyperparameter(self): return self.config["hyperparameter"] def test_config(self): if not isinstance(self.config, DeepDict): msg = "test_config failed, config is not of type DeepDict" LOG.error(msg) raise IOError(msg) sections = ["hyperparameter"] sections += [SETTINGSSOLVERPATH.split("/")[-1]] sections += [SETTINGSCUSTOMPATH.split("/")[-1]] sections_available = [True, True, True] for n, sec in enumerate(sections): if not self.config.has_section(sec): msg = "WARNING: config has no section {}".format(sec) LOG.warning(msg) sections_available[n] = False return sections_available def set_config(self, config): self.clear() if isinstance(config, dict): self.config = DeepDict() self.config.data = config elif isinstance(config, DeepDict): self.config = config else: msg = "unknown type ({}) for config passed, expected dict or DeepDict".format(type(config)) LOG.error(msg) raise IOError(msg) sections_available = self.test_config() if not sections_available[0]: msg = "Missing section {}".format("hyperparameter") LOG.error(msg) raise LookupError(msg) if not sections_available[1]: msg = "Missing section {}".format(SETTINGSSOLVERPATH) LOG.error(msg) raise LookupError(msg) else: try: self._extmembers += self.config.transfer_attrs(self, SETTINGSCUSTOMPATH.split("/")[-1]) except Exception as e: msg = "transfering custom section as class attributes failed, " \ "is the config path to your custom section correct? {}. Exception {}".format(SETTINGSCUSTOMPATH, e) LOG.error(msg) raise LookupError(msg) if sections_available[2]: try: self._extmembers += self.config.transfer_attrs(self, SETTINGSSOLVERPATH.split("/")[-1]) except Exception as e: msg = "transfering custom section as class attributes failed, " \ "is the config path to your custom section correct? {}. Exception {}".format(SETTINGSCUSTOMPATH, e) LOG.error(msg) raise LookupError(msg) return True def read_config(self, configfile): self.clear() self.configfilename = configfile self.config = DeepDict(configfile) sections_available = self.test_config() if not sections_available[0]: msg = "Missing section {}".format("hyperparameter") LOG.error(msg) raise LookupError(msg) if not sections_available[1]: msg = "Missing section {}".format(SETTINGSSOLVERPATH) LOG.error(msg) raise LookupError(msg) else: try: self._extmembers += self.config.transfer_attrs(self, SETTINGSSOLVERPATH.split("/")[-1]) except Exception as e: msg = "transfering custom section as class attributes failed, " \ "is the config path to your custom section correct? {}. Exception {}".format(SETTINGSSOLVERPATH, e) LOG.error(msg) raise LookupError(msg) if sections_available[2]: try: self._extmembers += self.config.transfer_attrs(self, SETTINGSCUSTOMPATH.split("/")[-1]) except Exception as e: msg = "transfering custom section as class attributes failed, " \ "is the config path to your custom section correct? {}. Exception {}".format(SETTINGSCUSTOMPATH, e) LOG.error(msg) raise LookupError(msg) return True def identifier(self, force=False): if self._identifier is None or force: self._identifier = datetime.datetime.now().strftime("%Y.%m.%d.%H.%M.%S") return self._identifier + + def register_member(self, name, value): + setattr(name, value) diff --git a/hyppopy/tests/test_projectmanager.py b/hyppopy/tests/test_projectmanager.py index 0e52fe1..61412e9 100644 --- a/hyppopy/tests/test_projectmanager.py +++ b/hyppopy/tests/test_projectmanager.py @@ -1,38 +1,135 @@ # DKFZ # # # Copyright (c) German Cancer Research Center, # Division of Medical and Biological Informatics. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE.txt or http://www.mitk.org for details. # # Author: Sven Wanner (s.wanner@dkfz.de) import os +import tempfile import unittest from hyppopy.projectmanager import ProjectManager DATA_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data") class ProjectManagerTestSuite(unittest.TestCase): def setUp(self): - pass + self.config = { + "hyperparameter": { + "C": { + "domain": "uniform", + "data": [0, 20], + "type": "float" + }, + "gamma": { + "domain": "uniform", + "data": [0.0001, 20.0], + "type": "float" + }, + "kernel": { + "domain": "categorical", + "data": ["linear", "sigmoid", "poly", "rbf"], + "type": "str" + }, + "decision_function_shape": { + "domain": "categorical", + "data": ["ovo", "ovr"], + "type": "str" + } + }, + "settings": { + "solver_plugin": { + "max_iterations": 300, + "use_plugin": "hyperopt", + "output_dir": os.path.join(tempfile.gettempdir(), 'results') + }, + "custom": { + "the_answer": 42 + } + }} - def test_attr_transfer(self): + def test_read_attrs(self): ProjectManager.read_config(os.path.join(DATA_PATH, *('Titanic', 'rf_config.xml'))) self.assertEqual(ProjectManager.data_name, 'train_cleaned.csv') self.assertEqual(ProjectManager.labels_name, 'Survived') self.assertEqual(ProjectManager.max_iterations, 3) self.assertEqual(ProjectManager.use_plugin, 'optunity') + hp = ProjectManager.get_hyperparameter() + self.assertTrue("n_estimators" in hp.keys()) + self.assertTrue("domain" in hp["n_estimators"].keys()) + self.assertTrue("data" in hp["n_estimators"].keys()) + self.assertTrue("type" in hp["n_estimators"].keys()) + self.assertEqual(hp["n_estimators"]["domain"], "uniform") + self.assertEqual(hp["n_estimators"]["type"], "int") + self.assertEqual(hp["n_estimators"]["data"], [3, 200]) + + self.assertTrue("max_depth" in hp.keys()) + self.assertTrue("domain" in hp["max_depth"].keys()) + self.assertTrue("data" in hp["max_depth"].keys()) + self.assertTrue("type" in hp["max_depth"].keys()) + self.assertEqual(hp["max_depth"]["domain"], "uniform") + self.assertEqual(hp["max_depth"]["type"], "int") + self.assertEqual(hp["max_depth"]["data"], [3, 50]) + + self.assertTrue("criterion" in hp.keys()) + self.assertTrue("domain" in hp["criterion"].keys()) + self.assertTrue("data" in hp["criterion"].keys()) + self.assertTrue("type" in hp["criterion"].keys()) + self.assertEqual(hp["criterion"]["domain"], "categorical") + self.assertEqual(hp["criterion"]["type"], "str") + self.assertEqual(hp["criterion"]["data"], ["gini", "entropy"]) + + def test_set_attrs(self): + self.assertTrue(ProjectManager.set_config(self.config)) + self.assertEqual(ProjectManager.max_iterations, 300) + self.assertEqual(ProjectManager.use_plugin, 'hyperopt') + self.assertEqual(ProjectManager.the_answer, 42) + + hp = ProjectManager.get_hyperparameter() + self.assertTrue("C" in hp.keys()) + self.assertTrue("domain" in hp["C"].keys()) + self.assertTrue("data" in hp["C"].keys()) + self.assertTrue("type" in hp["C"].keys()) + self.assertEqual(hp["C"]["domain"], "uniform") + self.assertEqual(hp["C"]["type"], "float") + self.assertEqual(hp["C"]["data"], [0, 20]) + + self.assertTrue("gamma" in hp.keys()) + self.assertTrue("domain" in hp["gamma"].keys()) + self.assertTrue("data" in hp["gamma"].keys()) + self.assertTrue("type" in hp["gamma"].keys()) + self.assertEqual(hp["gamma"]["domain"], "uniform") + self.assertEqual(hp["gamma"]["type"], "float") + self.assertEqual(hp["gamma"]["data"], [0.0001, 20.0]) + + self.assertTrue("kernel" in hp.keys()) + self.assertTrue("domain" in hp["kernel"].keys()) + self.assertTrue("data" in hp["kernel"].keys()) + self.assertTrue("type" in hp["kernel"].keys()) + self.assertEqual(hp["kernel"]["domain"], "categorical") + self.assertEqual(hp["kernel"]["type"], "str") + self.assertEqual(hp["kernel"]["data"], ["linear", "sigmoid", "poly", "rbf"]) + + self.assertTrue("decision_function_shape" in hp.keys()) + self.assertTrue("domain" in hp["decision_function_shape"].keys()) + self.assertTrue("data" in hp["decision_function_shape"].keys()) + self.assertTrue("type" in hp["decision_function_shape"].keys()) + self.assertEqual(hp["decision_function_shape"]["domain"], "categorical") + self.assertEqual(hp["decision_function_shape"]["type"], "str") + self.assertEqual(hp["decision_function_shape"]["data"], ["ovo", "ovr"]) + if __name__ == '__main__': unittest.main() diff --git a/hyppopy/tests/test_usecases.py b/hyppopy/tests/test_usecases.py new file mode 100644 index 0000000..0a80e6b --- /dev/null +++ b/hyppopy/tests/test_usecases.py @@ -0,0 +1,166 @@ +# DKFZ +# +# +# Copyright (c) German Cancer Research Center, +# Division of Medical and Biological Informatics. +# All rights reserved. +# +# This software is distributed WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. +# +# See LICENSE.txt or http://www.mitk.org for details. +# +# Author: Sven Wanner (s.wanner@dkfz.de) + +import os +import shutil +import unittest +import tempfile +import numpy as np +from sklearn.datasets import load_breast_cancer +from sklearn.model_selection import train_test_split + +from hyppopy.projectmanager import ProjectManager +from hyppopy.workflows.svc_usecase.svc_usecase import svc_usecase +from hyppopy.workflows.knc_usecase.knc_usecase import knc_usecase +from hyppopy.workflows.lda_usecase.lda_usecase import lda_usecase +from hyppopy.workflows.randomforest_usecase.randomforest_usecase import randomforest_usecase + + +DATA_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data") + + +class ProjectManagerTestSuite(unittest.TestCase): + + def setUp(self): + breast_cancer_data = load_breast_cancer() + x = breast_cancer_data.data + y = breast_cancer_data.target + x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.15, random_state=23) + + self.root = os.path.join(tempfile.gettempdir(), 'test_data') + if not os.path.isdir(self.root): + os.makedirs(self.root) + x_train_fname = os.path.join(self.root, 'x_train.npy') + y_train_fname = os.path.join(self.root, 'y_train.npy') + np.save(x_train_fname, x_train) + np.save(y_train_fname, y_train) + + self.test = [x_test, y_test] + self.config = { + "hyperparameter": {}, + "settings": { + "solver_plugin": { + "max_iterations": 5, + "use_plugin": "hyperopt", + "output_dir": os.path.join(self.root, 'test_results') + }, + "custom": { + "data_path": self.root, + "data_name": "x_train.npy", + "labels_name": "y_train.npy" + } + }} + + # def test_svc_usecase(self): + # hyperparameter = { + # "C": { + # "domain": "uniform", + # "data": [0.0001, 300.0], + # "type": "float" + # } + # } + # + # self.config["hyperparameter"] = hyperparameter + # ProjectManager.set_config(self.config) + # uc = svc_usecase() + # uc.run(save=True) + # res, best = uc.get_results() + # print(best) + + def test_randomforest_usecase(self): + hyperparameter = { + "n_estimators": { + "domain": "uniform", + "data": [1, 500], + "type": "int" + }, + "criterion": { + "domain": "categorical", + "data": ["gini", "entropy"], + "type": "str" + }, + "max_depth": { + "domain": "uniform", + "data": [1, 50], + "type": "int" + }, + "max_features": { + "domain": "categorical", + "data": ["auto", "sqrt", "log2"], + "type": "str" + } + } + + self.config["hyperparameter"] = hyperparameter + ProjectManager.set_config(self.config) + uc = randomforest_usecase() + uc.run(save=True) + res, best = uc.get_results() + print(best) + + # def test_lda_usecase(self): + # hyperparameter = { + # "solver": { + # "domain": "categorical", + # "data": ["svd", "lsqr", "eigen"], + # "type": "str" + # }, + # "tol": { + # "domain": "uniform", + # "data": [0.00000001, 1.0], + # "type": "float" + # } + # } + # + # self.config["hyperparameter"] = hyperparameter + # ProjectManager.set_config(self.config) + # uc = lda_usecase() + # uc.run(save=True) + # res, best = uc.get_results() + # print(best) + + def test_knc_usecase(self): + hyperparameter = { + "n_neighbors": { + "domain": "uniform", + "data": [1, 100], + "type": "int" + }, + "weights": { + "domain": "categorical", + "data": ["uniform", "distance"], + "type": "str" + }, + "algorithm": { + "domain": "categorical", + "data": ["auto", "ball_tree", "kd_tree", "brute"], + "type": "str" + } + } + + self.config["hyperparameter"] = hyperparameter + ProjectManager.set_config(self.config) + uc = knc_usecase() + uc.run(save=True) + res, best = uc.get_results() + print(best) + + def tearDown(self): + if os.path.isdir(self.root): + shutil.rmtree(self.root) + + +if __name__ == '__main__': + unittest.main()