diff --git a/.gitignore b/.gitignore index f6f552f..d8a17c5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,85 +1,87 @@ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class .pytest_cache/ +python_tests_xml + # C extensions *.so # Distribution / packaging .Python env/ build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ *.egg-info/ .installed.cfg *.egg .idea/ # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *,cover .hypothesis/ # Translations *.mo *.pot # Django stuff: *.log # Sphinx documentation docs/_build/ doc/ # PyBuilder target/ #Ipython Notebook .ipynb_checkpoints #Pycharm files *.iml # merging stuff *.orig *~ # Paths in repository mcml.py # images etc *.tif *.nrrd *.caffemodel # C++ stuff build* *.user diff --git a/bin/hyppopy_exe.py b/bin/hyppopy_exe.py index 086846b..a69d792 100644 --- a/bin/hyppopy_exe.py +++ b/bin/hyppopy_exe.py @@ -1,19 +1,23 @@ +# -*- coding: utf-8 -*- +# # DKFZ # +# # Copyright (c) German Cancer Research Center, # Division of Medical and Biological Informatics. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE.txt or http://www.mitk.org for details. +# +# Author: Sven Wanner (s.wanner@dkfz.de) -# -*- coding: utf-8 -*- from hyppopy.cmdtools import * if __name__ == '__main__': cmd_workflow() diff --git a/hyppopy/cmdtools.py b/hyppopy/cmdtools.py index c7bb8e5..15c0184 100644 --- a/hyppopy/cmdtools.py +++ b/hyppopy/cmdtools.py @@ -1,31 +1,33 @@ +# -*- coding: utf-8 -*- +# # DKFZ # +# # Copyright (c) German Cancer Research Center, # Division of Medical and Biological Informatics. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE.txt or http://www.mitk.org for details. - -# -*- coding: utf-8 -*- +# +# Author: Sven Wanner (s.wanner@dkfz.de) import argparse import logging LOG = logging.getLogger('hyppopy') from hyppopy.solver_factory import SolverFactory def cmd_workflow(): parser = argparse.ArgumentParser(description="") parser.add_argument('-v', '--verbosity', type=int, required=False, default=0, help='number of thoughts our thinker should produce') args_dict = vars(parser.parse_args()) - factory = SolverFactory.instance() diff --git a/hyppopy/solver/__init__.py b/hyppopy/deepdict/__init__.py similarity index 100% copy from hyppopy/solver/__init__.py copy to hyppopy/deepdict/__init__.py diff --git a/hyppopy/deepdict/deepdict.py b/hyppopy/deepdict/deepdict.py new file mode 100644 index 0000000..5aa031a --- /dev/null +++ b/hyppopy/deepdict/deepdict.py @@ -0,0 +1,375 @@ +# -*- coding: utf-8 -*- +# +# DKFZ +# +# +# Copyright (c) German Cancer Research Center, +# Division of Medical and Biological Informatics. +# All rights reserved. +# +# This software is distributed WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. +# +# See LICENSE.txt or http://www.mitk.org for details. +# +# Author: Sven Wanner (s.wanner@dkfz.de) + +import os +import json +import types +import pprint +import xmltodict +from dicttoxml import dicttoxml +from collections import OrderedDict + +import logging +LOG = logging.getLogger('hyppopy') + + +def convert_ordered2std_dict(obj): + """ + Helper function converting an OrderedDict into a standard lib dict. + :param obj: [OrderedDict] + """ + for key, value in obj.items(): + if isinstance(value, OrderedDict): + obj[key] = dict(obj[key]) + convert_ordered2std_dict(obj[key]) + + +def check_dir_existance(dirname): + """ + Helper function to check if a directory exists, creating it if not. + :param dirname: [str] full path of the directory to check + """ + if not os.path.exists(dirname): + os.mkdir(dirname) + + +class DeepDict(object): + """ + The DeepDict class represents a nested dictionary with additional functionality compared to a standard + lib dict. The data can be accessed and changed vie a pathlike access and dumped or read to .json/.xml files. + + Initializing instances using defaults creates an empty DeepDict. Using in_data enables to initialize the + object instance with data, where in_data can be a dict, or a filepath to a json or xml file. Using path sep + the appearance of path passing can be changed, a default data access via path would look like my_dd['target/section/path'] with path_sep='.' like so my_dd['target.section.path'] + + :param in_data: [dict] or [str], input dict or filename + :param path_sep: [str] path separator character + """ + _data = None + _sep = "/" + + def __init__(self, in_data=None, path_sep="/"): + """ + + """ + self.clear() + self._sep = path_sep + LOG.debug(f"path separator is: {self._sep}") + if in_data is not None: + if isinstance(in_data, str): + self.from_file(in_data) + elif isinstance(in_data, dict): + self.data = in_data + + def __str__(self): + """ + Enables print output for class instances, printing the instance data dict using pretty print + :return: [str] + """ + return pprint.pformat(self.data) + + def __eq__(self, other): + """ + Overloads the == operator comparing the instance data dictionaries for equality + :param other: [DeepDict] rhs + :return: [bool] + """ + return self.data == other.data + + def __getitem__(self, path): + """ + Overloads the return of the [] operator for data access. This enables access the DeepDict instance like so: + my_dd['target/section/path'] or my_dd[['target','section','path']] + :param path: [str] or [list(str)], the path to the target data structure level/content + :return: [object] + """ + return DeepDict.get_from_path(self.data, path, self.sep) + + def __setitem__(self, path, value=None): + """ + Overloads the setter for the [] operator for data assignment. + :param path: [str] or [list(str)], the path to the target data structure level/content + :param value: [object] rhs assignment object + """ + if isinstance(path, str): + path = path.split(self.sep) + if not isinstance(path, list) or isinstance(path, tuple): + raise IOError("Input Error, expect list[str] type for path") + if len(path) < 1: + raise IOError("Input Error, missing section strings") + + if not path[0] in self._data.keys(): + if value is not None and len(path) == 1: + self._data[path[0]] = value + else: + self._data[path[0]] = {} + + tmp = self._data[path[0]] + path.pop(0) + while True: + if len(path) == 0: + break + if path[0] not in tmp.keys(): + if value is not None and len(path) == 1: + tmp[path[0]] = value + else: + tmp[path[0]] = {} + tmp = tmp[path[0]] + else: + tmp = tmp[path[0]] + path.pop(0) + + def clear(self): + """ + clears the instance data + """ + LOG.debug("clear()") + self._data = {} + + def from_file(self, fname): + """ + Loads data from file. Currently implemented .json and .xml file reader. + :param fname: [str] filename + """ + if not isinstance(fname, str): + raise IOError("Input Error, expect str type for fname") + if fname.endswith(".json"): + self.read_json(fname) + elif fname.endswith(".xml"): + self.read_xml(fname) + else: + LOG.error("Unknown filetype, expect [.json, .xml]") + raise NotImplementedError("Unknown filetype, expect [.json, .xml]") + + def read_json(self, fname): + """ + Read json file + :param fname: [str] input filename + """ + if not isinstance(fname, str): + raise IOError("Input Error, expect str type for fname") + if not os.path.isfile(fname): + raise IOError(f"File {fname} not found!") + LOG.debug(f"read_json({fname})") + try: + with open(fname, "r") as read_file: + self._data = json.load(read_file) + DeepDict.value_traverse(self.data, callback=DeepDict.parse_type) + except Exception as e: + LOG.error(f"Error while reading json file {fname} or while converting types") + raise IOError("Error while reading json file {fname} or while converting types") + + def read_xml(self, fname): + """ + Read xml file + :param fname: [str] input filename + """ + if not isinstance(fname, str): + raise IOError("Input Error, expect str type for fname") + if not os.path.isfile(fname): + raise IOError(f"File {fname} not found!") + LOG.debug(f"read_xml({fname})") + try: + with open(fname, "r") as read_file: + xml = "".join(read_file.readlines()) + self._data = xmltodict.parse(xml, attr_prefix='') + DeepDict.value_traverse(self.data, callback=DeepDict.parse_type) + except Exception as e: + LOG.error(f"Error while reading xml file {fname} or while converting types") + raise IOError("Error while reading json file {fname} or while converting types") + + # if written with DeepDict, the xml contains a root node called + # deepdict which should beremoved for consistency reasons + if "deepdict" in self._data.keys(): + self._data = self._data["deepdict"] + self._data = dict(self.data) + # convert the orderes dict structure to a default dict for consistency reasons + convert_ordered2std_dict(self.data) + + def to_file(self, fname): + """ + Write to file, type is determined by checking the filename ending. + Currently implemented is writing to json and to xml. + :param fname: [str] filename + """ + if not isinstance(fname, str): + raise IOError("Input Error, expect str type for fname") + if fname.endswith(".json"): + self.write_json(fname) + elif fname.endswith(".xml"): + self.write_xml(fname) + else: + LOG.error(f"Unknown filetype, expect [.json, .xml]") + raise NotImplementedError("Unknown filetype, expect [.json, .xml]") + + def write_json(self, fname): + """ + Dump data to json file. + :param fname: [str] filename + """ + if not isinstance(fname, str): + raise IOError("Input Error, expect str type for fname") + check_dir_existance(os.path.dirname(fname)) + try: + LOG.debug(f"write_json({fname})") + with open(fname, "w") as write_file: + json.dump(self.data, write_file) + except Exception as e: + LOG.error(f"Failed dumping to json file: {fname}") + raise e + + def write_xml(self, fname): + """ + Dump data to json file. + :param fname: [str] filename + """ + if not isinstance(fname, str): + raise IOError("Input Error, expect str type for fname") + check_dir_existance(os.path.dirname(fname)) + xml = dicttoxml(self.data, custom_root='deepdict', attr_type=False) + LOG.debug(f"write_xml({fname})") + try: + with open(fname, "w") as write_file: + write_file.write(xml.decode("utf-8")) + except Exception as e: + LOG.error(f"Failed dumping to xml file: {fname}") + raise e + + @staticmethod + def get_from_path(data, path, sep="/"): + """ + Implements a nested dict access via a path like string like so path='target/section/path' + which is equivalent to my_dict['target']['section']['path']. + :param data: [dict] input dictionary + :param path: [str] pathlike string + :param sep: [str] path separator, default='/' + :return: [object] + """ + if not isinstance(data, dict): + LOG.error("Input Error, expect dict type for data") + raise IOError("Input Error, expect dict type for data") + if isinstance(path, str): + path = path.split(sep) + if not isinstance(path, list) or isinstance(path, tuple): + LOG.error(f"Input Error, expect list[str] type for path: {path}") + raise IOError("Input Error, expect list[str] type for path") + if not DeepDict.has_section(data, path[-1]): + LOG.error(f"Input Error, section {path[-1]} does not exist in dictionary") + raise IOError(f"Input Error, section {path[-1]} does not exist in dictionary") + for k in path: + data = data[k] + return data + + @staticmethod + def has_section(data, section, already_found=False): + """ + Checks if input dictionary has a key called section. The already_found parameter + is for internal recursion checks. + :param data: [dict] input dictionary + :param section: [str] key string to search for + :param already_found: recursion criteria check + :return: [bool] section found + """ + if not isinstance(data, dict): + LOG.error("Input Error, expect dict type for obj") + raise IOError("Input Error, expect dict type for obj") + if not isinstance(section, str): + LOG.error(f"Input Error, expect dict type for obj {section}") + raise IOError(f"Input Error, expect dict type for obj {section}") + if already_found: + return True + found = False + for key, value in data.items(): + if key == section: + found = True + if isinstance(value, dict): + found = DeepDict.has_section(data[key], section, found) + return found + + @staticmethod + def value_traverse(data, callback=None): + """ + Dictionary filter function, walks through the input dict (obj) calling the callback function for each value. + The callback function return is assigned the the corresponding dict value. + :param data: [dict] input dictionary + :param callback: + """ + if not isinstance(data, dict): + LOG.error("Input Error, expect dict type for obj") + raise IOError("Input Error, expect dict type for obj") + if not isinstance(callback, types.FunctionType): + LOG.error("Input Error, expect function type for callback") + raise IOError("Input Error, expect function type for callback") + for key, value in data.items(): + if isinstance(value, dict): + DeepDict.value_traverse(data[key], callback) + else: + data[key] = callback(value) + + @staticmethod + def parse_type(string): + """ + Type convert input string to float, int, list, tuple or string + :param string: [str] input string + :return: [T] converted output + """ + try: + a = float(string) + try: + b = int(string) + except ValueError: + return float(string) + if a == b: + return b + return a + except ValueError: + if string.startswith("[") and string.endswith("]"): + elements = string[1:-1].split(",") + li = [] + for e in elements: + li.append(DeepDict.parse_type(e)) + return li + elif string.startswith("(") and string.endswith(")"): + elements = string[1:-1].split(",") + li = [] + for e in elements: + li.append(DeepDict.parse_type(e)) + return tuple(li) + return string + + @property + def data(self): + return self._data + + @data.setter + def data(self, value): + if not isinstance(value, dict): + LOG.error(f"Input Error, expect dict type for value, but got {type(value)}") + raise IOError(f"Input Error, expect dict type for value, but got {type(value)}") + self.clear() + self._data = value + + @property + def sep(self): + return self._sep + + @sep.setter + def sep(self, value): + if not isinstance(value, str): + LOG.error(f"Input Error, expect str type for value, but got {type(value)}") + raise IOError(f"Input Error, expect str type for value, but got {type(value)}") + self._sep = value diff --git a/bin/hyppopy_exe.py b/hyppopy/helpers.py similarity index 81% copy from bin/hyppopy_exe.py copy to hyppopy/helpers.py index 086846b..8d9eca9 100644 --- a/bin/hyppopy_exe.py +++ b/hyppopy/helpers.py @@ -1,19 +1,16 @@ +# -*- coding: utf-8 -*- +# # DKFZ # +# # Copyright (c) German Cancer Research Center, # Division of Medical and Biological Informatics. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE.txt or http://www.mitk.org for details. - -# -*- coding: utf-8 -*- - -from hyppopy.cmdtools import * - - -if __name__ == '__main__': - cmd_workflow() +# +# Author: Sven Wanner (s.wanner@dkfz.de) diff --git a/hyppopy/isolver.py b/hyppopy/isolver.py index 41445d4..bc502c9 100644 --- a/hyppopy/isolver.py +++ b/hyppopy/isolver.py @@ -1,33 +1,42 @@ +# -*- coding: utf-8 -*- +# # DKFZ # +# # Copyright (c) German Cancer Research Center, # Division of Medical and Biological Informatics. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE.txt or http://www.mitk.org for details. +# +# Author: Sven Wanner (s.wanner@dkfz.de) -# -*- coding: utf-8 -*- - +import abc import logging LOG = logging.getLogger('hyppopy') -class ISolver(object): +class ISolver(object, metaclass=abc.ABCMeta): loss_function = None + space = None def set_loss_function(self, func): """ set loss function """ self.loss_function = func + def set_space(self, space): + """ + set loss function + """ + self.space = space + + @abc.abstractmethod def execute(self, *args, **kwargs): - raise NotImplementedError() + raise NotImplementedError('users must define execute to use this base class') - @property - def name(self): - return self.__name__ \ No newline at end of file diff --git a/hyppopy/ispace.py b/hyppopy/ispace.py new file mode 100644 index 0000000..356c259 --- /dev/null +++ b/hyppopy/ispace.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +# +# DKFZ +# +# +# Copyright (c) German Cancer Research Center, +# Division of Medical and Biological Informatics. +# All rights reserved. +# +# This software is distributed WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. +# +# See LICENSE.txt or http://www.mitk.org for details. +# +# Author: Sven Wanner (s.wanner@dkfz.de) + +import abc +import dpath +import dpath.util + +import logging +LOG = logging.getLogger('hyppopy') + + +class ISpace(object, metaclass=abc.ABCMeta): + _data = {} + + @abc.abstractmethod + def convert(self, *args, **kwargs): + raise NotImplementedError('users must define convert to use this base class') + + def clear(self): + LOG.debug("clear()") + self._data.clear() + + def get_section(self, path): + if isinstance(path, str): + try: + return dpath.util.get(self._data, path) + except Exception as e: + LOG.error(f"path mismatch exception: {e}") + raise IOError("path mismatch exception") + elif isinstance(path, list) or isinstance(path, tuple): + try: + return dpath.util.get(self._data, "/".join(path)) + except Exception as e: + LOG.error(f"path list content exception: {e}") + raise IOError("path list content exception") + else: + LOG.error("unknown path type") + raise IOError("unknown path type") + + def set(self, data): + LOG.debug(f"set({data})") + self._data = data + + def add_section(self, name, section=None): + LOG.debug(f"add_section({name}, {section})") + if section is None: + self._data[name] = {} + + def add_entry(self, name, value, section=None): + LOG.debug(f"add_entry({name}, {value}, {section})") + + def read_json(self, filename): + LOG.debug(f"read_json({filename})") + + def write_json(self, filename): + LOG.debug(f"write_json({filename})") + + def read_xml(self, filename): + LOG.debug(f"read_xml({filename})") + + def write_xml(self, filename): + LOG.debug(f"write_xml({filename})") + + def read(self, filename): + LOG.debug(f"read({filename})") + + def write(self, filename): + LOG.debug(f"write({filename})") diff --git a/bin/hyppopy_exe.py b/hyppopy/optimizer.py similarity index 63% copy from bin/hyppopy_exe.py copy to hyppopy/optimizer.py index 086846b..806f5a9 100644 --- a/bin/hyppopy_exe.py +++ b/hyppopy/optimizer.py @@ -1,19 +1,26 @@ +# -*- coding: utf-8 -*- +# # DKFZ # +# # Copyright (c) German Cancer Research Center, # Division of Medical and Biological Informatics. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE.txt or http://www.mitk.org for details. +# +# Author: Sven Wanner (s.wanner@dkfz.de) -# -*- coding: utf-8 -*- - -from hyppopy.cmdtools import * +from .isolver import ISolver +from .ispace import ISpace +class Optimizer(object): + _isolver = None + _ispace = None -if __name__ == '__main__': - cmd_workflow() + def __init__(self): + pass diff --git a/hyppopy/settings.py b/hyppopy/settings.py index 57fd76a..8ae7af6 100644 --- a/hyppopy/settings.py +++ b/hyppopy/settings.py @@ -1,34 +1,34 @@ # DKFZ # # Copyright (c) German Cancer Research Center, # Division of Medical and Biological Informatics. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE.txt or http://www.mitk.org for details. # -*- coding: utf-8 -*- import os import sys import logging ROOT = os.path.join(os.path.dirname(__file__), "..") LOGFILENAME = os.path.join(ROOT, 'logfile.log') -PLUGIN_DEFAULT_DIR = os.path.join(ROOT, *("hyppopy", "solver")) +PLUGIN_DEFAULT_DIR = os.path.join(ROOT, *("hyppopy", "solver_plugins")) sys.path.insert(0, ROOT) #LOG = logging.getLogger() logging.getLogger('hyppopy').setLevel(logging.DEBUG) logging.basicConfig(filename=LOGFILENAME, filemode='w', format='%(name)s - %(levelname)s - %(message)s') ''' LOG.debug('debug message') LOG.info('info message') LOG.warning('warning message') LOG.error('error message') LOG.critical('critical message') ''' diff --git a/hyppopy/solver_factory.py b/hyppopy/solver_factory.py index 025b155..8a6d0a7 100644 --- a/hyppopy/solver_factory.py +++ b/hyppopy/solver_factory.py @@ -1,76 +1,94 @@ +# -*- coding: utf-8 -*- +# # DKFZ # +# # Copyright (c) German Cancer Research Center, # Division of Medical and Biological Informatics. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE.txt or http://www.mitk.org for details. - -# -*- coding: utf-8 -*- +# +# Author: Sven Wanner (s.wanner@dkfz.de) from yapsy.PluginManager import PluginManager from .settings import PLUGIN_DEFAULT_DIR import logging LOG = logging.getLogger('hyppopy') class SolverFactory(object): _instance = None _plugin_dirs = [] _plugins = {} def __init__(self): if SolverFactory._instance is not None: pass else: LOG.debug("__init__()") SolverFactory._instance = self self.reset() self.load_plugins() - @staticmethod def instance(): """ Singleton instance access :return: [SolverFactory] instance """ LOG.debug("instance()") if SolverFactory._instance is None: SolverFactory() return SolverFactory._instance + def get_solver_names(self): + return list(self._plugins.keys()) + + def get_solver(self, name, **kwargs): + if name not in self._plugins.keys(): + LOG.error("Solver plugin name not available") + raise KeyError("Solver plugin name not available") + + if name == "HyperoptPlugin": + pass + elif name == "OptunityPlugin": + pass + else: + LOG.error("Solver plugin name does not match with key") + raise KeyError("Solver plugin name does not match with key") + def add_plugin_dir(self, dir): """ Add plugin directory """ LOG.debug(f"add_plugin_dir({dir})") self._plugin_dirs.append(dir) def reset(self): """ Reset solver factory """ LOG.debug("reset()") self._plugins = {} self._plugin_dirs = [] self.add_plugin_dir(PLUGIN_DEFAULT_DIR) def load_plugins(self): """ Load plugin modules from plugin paths """ LOG.debug("load_plugins()") LOG.debug(f"setPluginPlaces(" + " ".join(map(str, self._plugin_dirs))) manager = PluginManager() manager.setPluginPlaces(self._plugin_dirs) manager.collectPlugins() for plugin in manager.getAllPlugins(): - self._plugins[plugin.plugin_object.name] = plugin.plugin_object - LOG.info(f"Plugin: {plugin.plugin_object.name} loaded") + self._plugins[plugin.plugin_object.__class__.__name__] = plugin.plugin_object + LOG.info(f"Plugin: {plugin.plugin_object.__class__.__name__} loaded") diff --git a/hyppopy/solver/__init__.py b/hyppopy/solver_plugins/__init__.py similarity index 100% rename from hyppopy/solver/__init__.py rename to hyppopy/solver_plugins/__init__.py diff --git a/hyppopy/solver/hyperopt_plugin.py b/hyppopy/solver_plugins/hyperopt_plugin.py similarity index 83% rename from hyppopy/solver/hyperopt_plugin.py rename to hyppopy/solver_plugins/hyperopt_plugin.py index 660ba27..309f032 100644 --- a/hyppopy/solver/hyperopt_plugin.py +++ b/hyppopy/solver_plugins/hyperopt_plugin.py @@ -1,28 +1,33 @@ # DKFZ # # Copyright (c) German Cancer Research Center, # Division of Medical and Biological Informatics. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE.txt or http://www.mitk.org for details. # -*- coding: utf-8 -*- from yapsy.IPlugin import IPlugin import logging LOG = logging.getLogger('hyppopy') from hyppopy.isolver import ISolver +from hyppopy.ispace import ISpace -class HyperoptPlugin(IPlugin, ISolver): +class HyperoptSpace(ISpace): + + def convert(self): + pass - def __init__(self): - self.__name__ = "HyperoptPlugin" + +class HyperoptPlugin(IPlugin, ISolver): def execute(self, *args, **kwargs): - pass \ No newline at end of file + pass + diff --git a/hyppopy/solver/hyperopt_plugin.yapsy-plugin b/hyppopy/solver_plugins/hyperopt_plugin.yapsy-plugin similarity index 100% rename from hyppopy/solver/hyperopt_plugin.yapsy-plugin rename to hyppopy/solver_plugins/hyperopt_plugin.yapsy-plugin diff --git a/hyppopy/solver/optunity_plugin.py b/hyppopy/solver_plugins/optunity_plugin.py similarity index 97% rename from hyppopy/solver/optunity_plugin.py rename to hyppopy/solver_plugins/optunity_plugin.py index c04c509..7a53331 100644 --- a/hyppopy/solver/optunity_plugin.py +++ b/hyppopy/solver_plugins/optunity_plugin.py @@ -1,28 +1,28 @@ # DKFZ # # Copyright (c) German Cancer Research Center, # Division of Medical and Biological Informatics. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE.txt or http://www.mitk.org for details. # -*- coding: utf-8 -*- from yapsy.IPlugin import IPlugin import logging LOG = logging.getLogger('hyppopy') from hyppopy.isolver import ISolver class OptunityPlugin(IPlugin, ISolver): def __init__(self): self.__name__ = "OptunityPlugin" def execute(self, *args, **kwargs): - pass \ No newline at end of file + pass diff --git a/hyppopy/solver/optunity_plugin.yapsy-plugin b/hyppopy/solver_plugins/optunity_plugin.yapsy-plugin similarity index 100% rename from hyppopy/solver/optunity_plugin.yapsy-plugin rename to hyppopy/solver_plugins/optunity_plugin.yapsy-plugin diff --git a/hyppopy/tests/data/test_json.json b/hyppopy/tests/data/test_json.json new file mode 100644 index 0000000..8f58efb --- /dev/null +++ b/hyppopy/tests/data/test_json.json @@ -0,0 +1,26 @@ +{"widget": { + "debug": "on", + "window": { + "title": "Sample Konfabulator Widget", + "name": "main_window", + "width": 500, + "height": 500 + }, + "image": { + "src": "Images/Sun.png", + "name": "sun1", + "hOffset": 250, + "vOffset": 250, + "alignment": "center" + }, + "text": { + "data": "Click Here", + "size": 36, + "style": "bold", + "name": "text1", + "hOffset": 250, + "vOffset": 100, + "alignment": "center", + "onMouseUp": "sun1.opacity = (sun1.opacity / 100) * 90;" + } +}} \ No newline at end of file diff --git a/hyppopy/tests/data/test_paramset.json b/hyppopy/tests/data/test_paramset.json new file mode 100644 index 0000000..b481894 --- /dev/null +++ b/hyppopy/tests/data/test_paramset.json @@ -0,0 +1,22 @@ +{"parameter": { + "learningrate": { + "domain": "exponential", + "data": "(1e-5, 10.0)", + "type": "float" + }, + "layerdepth": { + "domain": "uniform", + "data": "[3,20]", + "type": "int" + }, + "normalize": { + "domain": "choice", + "data": "[0,1]", + "type": "int" + }, + "activation": { + "domain": "choice", + "data": "[ReLU,tanh,sigm,ELU]", + "type": "str" + } +}} \ No newline at end of file diff --git a/hyppopy/tests/data/test_xml.xml b/hyppopy/tests/data/test_xml.xml new file mode 100644 index 0000000..87b5c63 --- /dev/null +++ b/hyppopy/tests/data/test_xml.xml @@ -0,0 +1,22 @@ + + on + + main_window + 500 + 500 + + + 250 + 250 + center + + + text1 + 250 + 100 + center + + sun1.opacity = (sun1.opacity / 100) * 90; + + + \ No newline at end of file diff --git a/hyppopy/tests/test_deepdict.py b/hyppopy/tests/test_deepdict.py new file mode 100644 index 0000000..2d82f32 --- /dev/null +++ b/hyppopy/tests/test_deepdict.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- +# +# DKFZ +# +# +# Copyright (c) German Cancer Research Center, +# Division of Medical and Biological Informatics. +# All rights reserved. +# +# This software is distributed WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. +# +# See LICENSE.txt or http://www.mitk.org for details. +# +# Author: Sven Wanner (s.wanner@dkfz.de) +import os +import unittest + +from hyppopy.deepdict.deepdict import DeepDict + + +DATA_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data") + + +class DeepDictTestSuite(unittest.TestCase): + + def setUp(self): + self.test_data = { + 'widget': { + 'debug': 'on', + 'image': {'alignment': 'center', + 'hOffset': 250, + 'name': 'sun1', + 'src': 'Images/Sun.png', + 'vOffset': 250}, + 'text': {'alignment': 'center', + 'data': 'Click Here', + 'hOffset': 250, + 'name': 'text1', + 'onMouseUp': 'sun1.opacity = (sun1.opacity / 100) * 90;', + 'size': 36, + 'style': 'bold', + 'vOffset': 100}, + 'window': {'height': 500, + 'name': 'main_window', + 'title': 'Sample Konfabulator Widget', + 'width': 500} + } + } + + self.test_data2 = {"test": { + "section": { + "var1": 100, + "var2": 200 + } + }} + + def test_fileIO(self): + dd_json = DeepDict(os.path.join(DATA_PATH, 'test_json.json')) + dd_xml = DeepDict(os.path.join(DATA_PATH, 'test_xml.xml')) + dd_dict = DeepDict(self.test_data) + + self.assertTrue(list(self.test_data.keys())[0] == list(dd_json.data.keys())[0]) + self.assertTrue(list(self.test_data.keys())[0] == list(dd_xml.data.keys())[0]) + self.assertTrue(list(self.test_data.keys())[0] == list(dd_dict.data.keys())[0]) + for key in self.test_data['widget'].keys(): + self.assertTrue(self.test_data['widget'][key] == dd_json.data['widget'][key]) + self.assertTrue(self.test_data['widget'][key] == dd_xml.data['widget'][key]) + self.assertTrue(self.test_data['widget'][key] == dd_dict.data['widget'][key]) + for key in self.test_data['widget'].keys(): + if key == 'debug': + self.assertTrue(dd_json.data['widget']["debug"] == "on") + self.assertTrue(dd_xml.data['widget']["debug"] == "on") + self.assertTrue(dd_dict.data['widget']["debug"] == "on") + else: + for key2, value2 in self.test_data['widget'][key].items(): + self.assertTrue(value2 == dd_json.data['widget'][key][key2]) + self.assertTrue(value2 == dd_xml.data['widget'][key][key2]) + self.assertTrue(value2 == dd_dict.data['widget'][key][key2]) + + dd_dict.to_file(os.path.join(DATA_PATH, 'write_to_json_test.json')) + dd_dict.to_file(os.path.join(DATA_PATH, 'write_to_xml_test.xml')) + self.assertTrue(os.path.isfile(os.path.join(DATA_PATH, 'write_to_json_test.json'))) + self.assertTrue(os.path.isfile(os.path.join(DATA_PATH, 'write_to_xml_test.xml'))) + dd_json = DeepDict(os.path.join(DATA_PATH, 'write_to_json_test.json')) + dd_xml = DeepDict(os.path.join(DATA_PATH, 'write_to_xml_test.xml')) + self.assertTrue(dd_json == dd_dict) + self.assertTrue(dd_xml == dd_dict) + try: + os.remove(os.path.join(DATA_PATH, 'write_to_json_test.json')) + os.remove(os.path.join(DATA_PATH, 'write_to_xml_test.xml')) + except Exception as e: + print(e) + print("Warning: Failed to delete temporary data during tests!") + + def test_has_section(self): + dd = DeepDict(self.test_data) + self.assertTrue(DeepDict.has_section(dd.data, 'hOffset')) + self.assertTrue(DeepDict.has_section(dd.data, 'window')) + self.assertTrue(DeepDict.has_section(dd.data, 'widget')) + self.assertFalse(DeepDict.has_section(dd.data, 'notasection')) + + def test_data_access(self): + dd = DeepDict(self.test_data) + self.assertEqual(dd['widget/window/height'], 500) + self.assertEqual(dd['widget/image/name'], 'sun1') + self.assertTrue(isinstance(dd['widget/window'], dict)) + self.assertEqual(len(dd['widget/window']), 4) + + dd = DeepDict(path_sep=".") + dd.data = self.test_data + self.assertEqual(dd['widget.window.height'], 500) + self.assertEqual(dd['widget.image.name'], 'sun1') + self.assertTrue(isinstance(dd['widget.window'], dict)) + self.assertEqual(len(dd['widget.window']), 4) + + def test_data_adding(self): + dd = DeepDict() + dd["test/section/var1"] = 100 + dd["test/section/var2"] = 200 + self.assertTrue(dd.data == self.test_data2) + + dd = DeepDict() + dd["test"] = {} + dd["test/section"] = {} + dd["test/section/var1"] = 100 + dd["test/section/var2"] = 200 + self.assertTrue(dd.data == self.test_data2) + + def test_sample_space(self): + dd = DeepDict(os.path.join(DATA_PATH, 'test_paramset.json')) + self.assertEqual(len(dd[['parameter', 'activation', 'data']]), 4) + self.assertEqual(dd['parameter/activation/data'], ['ReLU', 'tanh', 'sigm', 'ELU']) + self.assertTrue(isinstance(dd['parameter/activation/data'], list)) + self.assertTrue(isinstance(dd['parameter/activation/data'][0], str)) + self.assertEqual(dd['parameter/layerdepth/data'], [3, 20]) + self.assertTrue(isinstance(dd['parameter/layerdepth/data'], list)) + self.assertTrue(isinstance(dd['parameter/layerdepth/data'][0], int)) + self.assertTrue(isinstance(dd['parameter/learningrate/data'][0], float)) + self.assertEqual(dd['parameter/learningrate/data'][0], 1e-5) + self.assertEqual(dd['parameter/learningrate/data'][1], 10.0) + + +if __name__ == '__main__': + unittest.main() diff --git a/bin/hyppopy_exe.py b/hyppopy/tests/test_helpers.py similarity index 66% copy from bin/hyppopy_exe.py copy to hyppopy/tests/test_helpers.py index 086846b..364f6ca 100644 --- a/bin/hyppopy_exe.py +++ b/hyppopy/tests/test_helpers.py @@ -1,19 +1,30 @@ +# -*- coding: utf-8 -*- +# # DKFZ # +# # Copyright (c) German Cancer Research Center, # Division of Medical and Biological Informatics. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE.txt or http://www.mitk.org for details. +# +# Author: Sven Wanner (s.wanner@dkfz.de) -# -*- coding: utf-8 -*- +import unittest + +from hyppopy.helpers import * + + +class HelpersTestSuite(unittest.TestCase): -from hyppopy.cmdtools import * + def setUp(self): + pass if __name__ == '__main__': - cmd_workflow() + unittest.main() diff --git a/hyppopy/tests/test_ispace.py b/hyppopy/tests/test_ispace.py new file mode 100644 index 0000000..ca6b42a --- /dev/null +++ b/hyppopy/tests/test_ispace.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# +# DKFZ +# +# +# Copyright (c) German Cancer Research Center, +# Division of Medical and Biological Informatics. +# All rights reserved. +# +# This software is distributed WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. +# +# See LICENSE.txt or http://www.mitk.org for details. +# +# Author: Sven Wanner (s.wanner@dkfz.de) + +import unittest + +from hyppopy.ispace import ISpace + + +class TestSpace(ISpace): + def convert(self, *args, **kwargs): + pass + + +class ISpaceTestSuite(unittest.TestCase): + + def setUp(self): + pass + + def test_IO(self): + ispace = TestSpace() + tdict = { + "a": { + "b": { + "3": 2, + "43": 30, + "c": [], + "d": ['red', 'buggy', 'bumpers'], + } + }, + "A": {"X": 1} + } + ispace.set(tdict) + self.assertEqual(ispace.get_section('a/b/d')[0], 'red') + + +if __name__ == '__main__': + unittest.main() diff --git a/hyppopy/tests/test_pluginmechanism.py b/hyppopy/tests/test_solver_factory.py similarity index 78% rename from hyppopy/tests/test_pluginmechanism.py rename to hyppopy/tests/test_solver_factory.py index c5ed9c6..417c246 100644 --- a/hyppopy/tests/test_pluginmechanism.py +++ b/hyppopy/tests/test_solver_factory.py @@ -1,35 +1,35 @@ +# -*- coding: utf-8 -*- +# # DKFZ # +# # Copyright (c) German Cancer Research Center, # Division of Medical and Biological Informatics. # All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE.txt or http://www.mitk.org for details. - -# -*- coding: utf-8 -*- +# +# Author: Sven Wanner (s.wanner@dkfz.de) import unittest from hyppopy.solver_factory import SolverFactory class PluginMechanismTestSuite(unittest.TestCase): - """Advanced test cases.""" def setUp(self): pass - def test_1(self): - """ - - :return: - """ + def test_factory_build(self): factory = SolverFactory.instance() + print(factory.get_solver_names()) + factory.get_solver("HyperoptPlugin") if __name__ == '__main__': unittest.main() diff --git a/requirements.txt b/requirements.txt index 31ec439..f1f0aae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,15 @@ # sphinx is used for building the documentation sphinx # xmlrunner is used in tests.py and needed to get nicely formatted output for # jenkins to interpret. xmlrunner # yapsy is a plugin manager package yapsy -# you can also specify specific version here, as e.g. -# Framework==0.9.4 -# Library>=0.2 +# xmltodict is a package for converting xml structure to nested dicts +xmltodict + +# dicttoxml is a package for converting nested dicts to xml structure +dicttoxml