diff --git a/evaluation/create_algorithm_performances.py b/evaluation/create_algorithm_performances.py
new file mode 100644
index 0000000..bc1ba3a
--- /dev/null
+++ b/evaluation/create_algorithm_performances.py
@@ -0,0 +1,135 @@
+import os
+import numpy as np
+import pandas as pd
+from glob import glob
+
+from joblib import Parallel, delayed
+from tqdm import tqdm
+
+from evaluation.evaluation_settings import Teams
+from statistics.dice_calculations import compute_dice_coefficient_per_instance
+from plots.example_image import InstanceImage
+
+
+def find_files(path, pattern):
+ files = []
+ print("Finding files")
+ for dir, _, _ in os.walk(path):
+ files.extend(glob(os.path.join(dir, pattern)))
+ return files
+
+
+def load_image(path, empty_image_dim=[540, 960]):
+
+ if "human" in path:
+ image = InstanceImage(path, "", do_cleaning=True).get_image()
+ else:
+ # load image
+ if os.path.isfile(path):
+ image = InstanceImage(path, "", do_cleaning=False)
+
+ else: # otherwise create empty image
+ image = np.zeros(empty_image_dim).astype('int')
+
+ # create output dict
+ output = dict(
+ path=path,
+ image=image
+ )
+ return output
+
+
+def participant_to_groundtruth(path_participant):
+ global path_groundtruth, pattern_participants, pattern_groundtruth, path_participants
+ image_path = path_participant.replace(path_participants, "").split("/")[-5:]
+ image_path = "/".join(image_path).replace(pattern_participants, pattern_groundtruth)
+ image_path = os.path.join(path_groundtruth, image_path)
+ return image_path
+
+if __name__ == "__main__":
+ path_groundtruth = "/media/rosst/ssd_2tb/datasets/ROBUST-MIS/Testing"
+ path_participants = "/media/rosst/ssd_2tb/datasets/ROBUST-MIS/Results/9614273"
+ pattern_groundtruth = "instrument_instances.png"
+ pattern_participants = "output.png"
+ file_results = "saved_all_image_performances.pkl"
+
+ try:
+ df = pd.read_pickle(file_results)
+ print(f"Loading file {file_results} was successful")
+ except:
+ print(f"Loading file {file_results} was not successful, create results")
+
+ # get groundtruth files
+ files_groundtruth = find_files(path_groundtruth, pattern=pattern_groundtruth)
+ files_participants = find_files(path_participants, pattern=pattern_participants)
+
+ # load files
+ files_to_load = files_groundtruth + files_participants
+ files_loaded = Parallel(n_jobs=12)(delayed(load_image)(file) for file in tqdm(files_to_load))
+
+ # convert to a dict
+ images = dict()
+ for file in files_loaded:
+ images[file["path"]] = file["image"]
+ del files_loaded
+
+ # participant image to groundtruth
+ comparison_pairs = list()
+ for file in files_participants:
+ gt_path = participant_to_groundtruth(file)
+ if gt_path in images:
+ prediction = images[file]
+ gt = images[gt_path]
+ comparison_pairs.append(dict(
+ mask_pred=prediction,
+ mask_gt=gt,
+ path_gt=gt_path,
+ path_pred=file,
+ ))
+
+
+ def get_n_instances(mask_gt, mask_pred):
+ n_instances_gt = len(np.unique(mask_gt)) - 1,
+ n_instances_pred = len(np.unique(mask_pred)) - 1,
+ return dict(n_instances_gt=n_instances_gt, n_instances_pred=n_instances_pred)
+
+
+ # calculate dice values
+ dce_values = Parallel(n_jobs=12)(
+ delayed(compute_dice_coefficient_per_instance)(pair["mask_gt"], pair["mask_pred"]) for pair in
+ tqdm(comparison_pairs))
+ n_instances = Parallel(n_jobs=12)(
+ delayed(get_n_instances)(pair["mask_gt"], pair["mask_pred"]) for pair in tqdm(comparison_pairs))
+
+ # images to instances
+ values_to_images = list()
+ dce_values_counter = 0
+ for file in tqdm(files_participants):
+ gt_path = participant_to_groundtruth(file)
+ if gt_path in images:
+ keys = list(dce_values[dce_values_counter].keys())
+ instruments = [key for key in keys if "instrument" in key]
+ instance_dce_values = list(dce_values[dce_values_counter].values())
+ mi_dsc = np.array(instance_dce_values)[1:].mean()
+
+ team, stage, or_type, or_nr, frame_nr = file.replace(path_participants, "").split("/")[1:-1]
+ team = Teams.get_team_by_synid(team)
+ values_to_images.append(dict(
+ file_gt=gt_path,
+ file_participant=file,
+ team=team,
+ stage=stage,
+ or_type=or_type,
+ or_nr=or_nr,
+ frame_nr=frame_nr,
+ MI_DSC=mi_dsc,
+ dce_values_raw=dce_values[dce_values_counter],
+ **n_instances[dce_values_counter]
+ ))
+ dce_values_counter += 1
+
+ # to pandas dataframe
+ df = pd.DataFrame(values_to_images)
+ df["n_instances_gt"] = df["n_instances_gt"].apply(lambda x: x[0])
+ df["n_instances_pred"] = df["n_instances_pred"].apply(lambda x: x[0])
+ df.to_pickle("saved_all_image_performances.pkl")
\ No newline at end of file
diff --git a/evaluation/evaluation_settings.py b/evaluation/evaluation_settings.py
new file mode 100644
index 0000000..b8f6c0a
--- /dev/null
+++ b/evaluation/evaluation_settings.py
@@ -0,0 +1,67 @@
+from enum import Enum, auto
+
+
+class EVALUATION_SETTINGS:
+ PATH_TESTING_DATA = "/media/rosst/ssd_2tb/datasets/ROBUST-MIS/Testing"
+ PATH_OUTPUT_FOLDER = "/media/rosst/ssd_2tb/datasets/ROBUST-MIS/Results"
+ PATH_TRAINING_DATA = "/media/rosst/hdd_data/datasets/robustmis2019/release/instance_segmentation/release_instance_segmentation_30092019/Training"
+ PATH_SHELL_SCRIPT = "/media/rosst/hdd_data/development/robust_mis_2019/evaluation/run_docker.sh"
+ DOCKER_MAX_EXECUTION_TIME_IN_S = 8 * 60 * 60 # 8h
+ MOCKUP = False
+
+
+class Testcase(Enum):
+ BINARY_SEGMENTATION = 9614245
+ MULTIPLE_INSTANCE_DETECTION = 9614272
+ MULTIPLE_INSTANCE_SEGMENTATION = 9614273
+
+ @staticmethod
+ def string_to_enum(input_str):
+ if not isinstance(input_str, str):
+ output = input_str
+ else:
+ output = eval(input_str)
+ return output
+
+
+class Teams:
+ TEAMS = {
+ "syn20824431": "fisensee",
+ "syn20824432": "fisensee",
+ "syn20821332": "NCT",
+ "syn20812165": "Djh",
+ "syn20821674": "Djh",
+ "syn20821710": "Djh",
+ "syn20819950": "www",
+ "syn20812156": "www",
+ "syn20784332": "www",
+ "syn20805359": "CASIA_SRL",
+ "syn20812106": "CASIA_SRL",
+ "syn20806431": "Prometheus",
+ "syn20820446": "Uniandes",
+ "syn20820448": "Uniandes",
+ "syn20820447": "Uniandes",
+ "syn20821333": "Raabid",
+ "syn20758210": "Caresyntax",
+ "syn20814896": "Caresyntax",
+ "syn20814856": "Caresyntax",
+ "syn20818289": "VIE",
+ "syn20818338": "VIE",
+ "syn20818337": "VIE",
+ "syn20780161": "SQUASH",
+ "syn20780293": "SQUASH",
+ "syn20780295": "SQUASH",
+ "syn20823093": "haoyun",
+
+
+ }
+
+ @staticmethod
+ def get_team_by_synid(syn_id):
+ syn_id = syn_id.replace("docker_synapse_org_", "")
+ syn_id = syn_id.split("_")[0]
+ try:
+ team_name = Teams.TEAMS[syn_id]
+ except KeyError:
+ team_name = syn_id
+ return team_name
diff --git a/plots/example_image.py b/plots/example_image.py
new file mode 100644
index 0000000..e51ceea
--- /dev/null
+++ b/plots/example_image.py
@@ -0,0 +1,291 @@
+import os
+import numpy as np
+
+import cv2
+from imageio import imread
+from scipy.ndimage import binary_dilation
+
+from evaluation.evaluation_settings import EVALUATION_SETTINGS, Testcase, Teams
+from statistics.dice_calculations import compute_dice_coefficient_per_instance, compute_dice_coefficient
+
+
+class ExampleImage:
+
+ def __init__(self, image_path, task, human_flag="human"):
+ # define variables
+ self.image_path = image_path
+ self.participants = ParticipantImages(image_path, task, human_flag=human_flag)
+ self.image_contours = []
+ self.colors = []
+
+ # settings
+ self.contour_size = 5
+
+ # run init functions
+ self.read_images()
+ self.define_contour_colors()
+
+ def define_contour_colors(self):
+ colors = [
+ (0, 255, 0),
+ (255, 20, 147),
+ (255, 255, 0),
+ (0, 255, 255),
+ (153, 101, 21),
+ (102, 102, 0),
+ (204, 255, 229),
+ (160, 160, 160),
+ (0, 128, 255),
+ (255, 153, 255),
+ ]
+ self.colors = colors
+
+ def read_images(self):
+ # create paths
+ path_img_instances = os.path.join(EVALUATION_SETTINGS.PATH_TESTING_DATA, self.image_path,
+ "instrument_instances.png")
+ path_img_raw = os.path.join(EVALUATION_SETTINGS.PATH_TESTING_DATA, self.image_path, "raw.png")
+
+ # read images
+ self.image_raw = imread(path_img_raw)
+ self.image_raw = np.array(self.image_raw)
+ self.image_instances = InstanceImage(path_img_instances, "groundtruth")
+
+ def create_image_with_contours(self):
+ # create image
+ image = self.image_raw.copy()
+
+ # get contours
+ contours = self.image_instances.get_contours()
+
+ # draw contours
+ for i, contour in enumerate(contours):
+ cv2.drawContours(image, contour, -1, self.colors[i], self.contour_size)
+
+ return image
+
+ def save_image_with_contours(self, path):
+ contour_image = self.create_image_with_contours()
+ contour_image.imsave(path, contour_image)
+
+ def get_statistics(self):
+ participant_images = self.participants.get_images()
+ participant_statistics = []
+ for participant_prediction_img in participant_images:
+ # get image info
+ image_info = dict(
+ path_pred=participant_prediction_img.path,
+ path_gt=self.image_instances.path,
+ team=participant_prediction_img.get_name()
+ )
+
+ # get images
+ participant_prediction_img = participant_prediction_img.image_instances
+ ground_truth_img = self.image_instances.image_instances
+
+ # CASE BINARY SEGMENTATION
+ ground_truth_img_binary_seg = np.copy(ground_truth_img)
+ participant_prediction_img_binary_seg = np.copy(participant_prediction_img)
+
+ ground_truth_img_binary_seg[ground_truth_img_binary_seg > 0] = 1
+ participant_prediction_img_binary_seg[participant_prediction_img_binary_seg > 0] = 1
+
+ dice_coefficient = \
+ compute_dice_coefficient(mask_gt=ground_truth_img_binary_seg,
+ mask_pred=participant_prediction_img_binary_seg)
+
+ # multiple instance dice
+ instrument_dices = compute_dice_coefficient_per_instance(ground_truth_img,
+ participant_prediction_img)
+ instrument_list = list(instrument_dices.keys())
+ instrument_list.remove("background")
+ dice_instances = np.array([instrument_dices[key] for key in instrument_list])
+
+ statistics = dict(
+ instance_dices_coefficients=dice_instances,
+ dice_coefficient = [dice_coefficient],
+ **image_info
+ )
+
+ participant_statistics.append(statistics)
+ return participant_statistics
+
+ def get_participant_images(self):
+ return self.participants.get_images()
+
+ def get_participant_team_names(self):
+ return sorted(self.participants.get_team_names())
+
+
+class InstanceImage:
+ VALID_INSTRUMENT_AREA = 0.03
+
+ def __init__(self, path, name, do_cleaning=False):
+ self.name = name
+ self.path = path
+ self.image_instances = None
+ self.image_contours = []
+ self.empty_image_dim = [540, 960]
+ self.do_cleaning = do_cleaning
+
+ self.load_object()
+
+ def load_object(self):
+ self.imload()
+ self.make_instance_nr_continues()
+ self.calculate_contours()
+ self.calculate_area_of_instances()
+
+ if self.do_cleaning:
+ self.remove_small_parts()
+
+ # update contours and areas
+ self.calculate_contours()
+ self.calculate_area_of_instances()
+
+ def imload(self):
+ # if image exists load image
+ if os.path.isfile(self.path):
+ output = imread(self.path)
+
+ else: # otherwise create empty image
+ output = np.zeros(self.empty_image_dim).astype('int')
+
+ self.image_instances = output
+
+ def get_instances(self):
+ instances = np.unique(self.image_instances)
+ instances = instances[1:] # remove background
+ return instances
+
+ def get_number_of_instances(self):
+ n_instances = len(self.get_instances())
+ return n_instances
+
+ def calculate_area_of_instances(self):
+ self.instance_areas = []
+ n_instances = np.unique(self.image_instances)
+ n_instances = n_instances[1:] # remove background
+
+ for current_instance_nr in n_instances:
+ # create current binary image
+ current_instance = np.copy(self.image_instances)
+ current_instance[self.image_instances == current_instance_nr] = 1
+ current_instance[self.image_instances != current_instance_nr] = 0
+ instance_area = np.sum(current_instance)
+ self.instance_areas.append(instance_area)
+
+ def calculate_contours(self):
+ self.image_contours = []
+ n_instances = np.unique(self.image_instances)
+ n_instances = n_instances[1:] # remove background
+
+ for current_instance_nr in n_instances:
+ # create current binary image
+ current_instance = np.copy(self.image_instances)
+ current_instance[self.image_instances == current_instance_nr] = 1
+ current_instance[self.image_instances != current_instance_nr] = 0
+
+ # get contours
+ # imgray = cv2.cvtColor(current_instance, cv2.COLOR_BGR2GRAY)
+ # ret, thresh = cv2.threshold(imgray, 127, 255, 0)
+ contours, _ = cv2.findContours(current_instance, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
+ self.image_contours.append(contours)
+
+ def get_contours(self):
+ return self.image_contours
+
+ def get_area_of_instances(self):
+ return self.instance_areas
+
+ def get_image(self):
+ return np.array(self.image_instances)
+
+ def get_name(self):
+ return self.name
+
+ def make_instance_nr_continues(self):
+ # get instance numbers
+ instances = self.get_instances()
+
+ # make instances continues
+ for instance_nr_new, instance in enumerate(instances):
+ self.image_instances[self.image_instances == instance] = instance_nr_new + 1 # because 0 is bg
+
+ def remove_small_parts(self):
+ self.image_raw = self.image_instances.copy()
+
+ # get areas of all instances
+ instance_areas = np.array(self.get_area_of_instances())
+ if self.get_number_of_instances()>1:
+
+ # get instances
+ instances = self.get_instances()
+
+ # calculate percentage of all instances with respect to the biggest one
+ areas_percentages = instance_areas / np.max(instance_areas)
+
+ # remove regions that are smaller than 3% of the biggest area
+ small_regions_idx = np.where(areas_percentages<InstanceImage.VALID_INSTRUMENT_AREA)[0]
+ if len(small_regions_idx>0):
+ for small_region_idx in small_regions_idx:
+ # get current instance nr of small region
+ current_idx = instances[small_region_idx]
+ mask_image = self.image_instances==current_idx
+
+ # find connected regions
+ dilated_instance = binary_dilation(mask_image)
+ idx_in_element = np.unique(self.image_instances[dilated_instance])[1:]
+
+ if len(idx_in_element)>1:
+ # get biggest connected region if it exists
+ idx_biggest_area = np.argmax(instance_areas[idx_in_element-1])
+ new_instance_nr = instances[idx_biggest_area]
+ else:
+ # if no biggest connected region exists, merge into bg
+ new_instance_nr = 0
+
+ # set new instance nr
+ self.image_instances[self.image_instances==current_idx] = new_instance_nr
+
+
+class ParticipantImages:
+
+ def __init__(self, image, task, human_flag = "human"):
+ self.image = image
+ self.task = Testcase.string_to_enum(task)
+ self.folder_participants = []
+ self.read_images()
+
+ def get_participant_files(self):
+ evaluation_folder = os.path.join(EVALUATION_SETTINGS.PATH_OUTPUT_FOLDER, str(self.task.value))
+ participants = os.listdir(evaluation_folder)
+ participant_images = []
+ for participant in participants:
+ if ".csv" not in participant:
+ path_participant = os.path.join(evaluation_folder, participant)
+ path_participant_img = os.path.join(path_participant, self.image, "output.png")
+ participant_images.append(dict(file=path_participant_img, participant=participant))
+ return participant_images
+
+ def read_images(self):
+ paths_participant_image = self.get_participant_files()
+ participant_images = []
+ for participant_path in paths_participant_image:
+ if "human" in participant_path["file"]:
+ participant_img = InstanceImage(participant_path["file"], participant_path["participant"],
+ do_cleaning=True)
+ else:
+ participant_img = InstanceImage(participant_path["file"], participant_path["participant"])
+ participant_images.append(participant_img)
+
+ self.participant_images = participant_images
+
+ def get_images(self):
+ return self.participant_images
+
+ def get_team_names(self):
+ evaluation_folder = os.path.join(EVALUATION_SETTINGS.PATH_OUTPUT_FOLDER, str(self.task.value))
+ participants = os.listdir(evaluation_folder)
+ participants_names = [Teams.get_team_by_synid(participant) for participant in participants]
+ return participants_names
diff --git a/statistics/__init__.py b/statistics/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/evaluation/dice_calculations.py b/statistics/dice_calculations.py
similarity index 97%
rename from evaluation/dice_calculations.py
rename to statistics/dice_calculations.py
index e28bd79..84174be 100644
--- a/evaluation/dice_calculations.py
+++ b/statistics/dice_calculations.py
@@ -1,94 +1,98 @@
import numpy as np
from scipy.optimize import linear_sum_assignment as hungarian_algorithm
def compute_dice_coefficient(mask_gt, mask_pred):
"""Compute soerensen-dice coefficient.
compute the soerensen-dice coefficient between the ground truth mask `mask_gt`
and the predicted mask `mask_pred`.
Args:
mask_gt: 3-dim Numpy array of type bool. The ground truth mask.
mask_pred: 3-dim Numpy array of type bool. The predicted mask.
Returns:
the dice coeffcient as float. If both masks are empty, the result is NaN
"""
+ # binarize image
+ mask_gt[mask_gt>0] = 1
+ mask_pred[mask_pred>0] = 1
+ # calculate volume
volume_sum = mask_gt.sum() + mask_pred.sum()
if volume_sum == 0:
return np.NaN
volume_intersect = (mask_gt & mask_pred).sum()
return 2 * volume_intersect / volume_sum
def compute_dice_coefficient_per_instance(mask_gt, mask_pred):
"""Compute instance soerensen-dice coefficient.
compute the soerensen-dice coefficient between the ground truth mask `mask_gt`
and the predicted mask `mask_pred` for multiple instances.
Args:
mask_gt: 3-dim Numpy array of type int. The ground truth image, where 0 means background and 1-N is an
instrument instance.
mask_pred: 3-dim Numpy array of type int. The predicted mask, where 0 means background and 1-N is an
instrument instance.
Returns:
a instance dictionary with the dice coeffcient as float.
"""
# get number of labels in image
instances_gt = np.unique(mask_gt)
instances_pred = np.unique(mask_pred)
# create performance matrix
performance_matrix = np.zeros((len(instances_gt), len(instances_pred)))
masks = []
# calculate dice score for each ground truth to predicted instance
for counter_gt, instance_gt in enumerate(instances_gt):
# create binary mask for current gt instance
gt = mask_gt.copy()
gt[mask_gt != instance_gt] = 0
gt[mask_gt == instance_gt] = 1
masks_row = []
for counter_pred, instance_pred in enumerate(instances_pred):
# make binary mask for current predicted instance
prediction = mask_pred.copy()
prediction[mask_pred != instance_pred] = 0
prediction[mask_pred == instance_pred] = 1
# calculate dice
performance_matrix[counter_gt, counter_pred] = compute_dice_coefficient(gt, prediction)
masks_row.append([gt, prediction])
masks.append(masks_row)
# assign instrument instances according to hungarian algorithm
label_assignment = hungarian_algorithm(performance_matrix * -1)
label_nr_gt, label_nr_pred = label_assignment
# get performance per instance
image_performance = []
for i in range(len(label_nr_gt)):
instance_dice = performance_matrix[label_nr_gt[i], label_nr_pred[i]]
image_performance.append(instance_dice)
missing_pred = np.absolute(len(instances_pred) - len(image_performance))
missing_gt = np.absolute(len(instances_gt) - len(image_performance))
n_missing = np.max([missing_gt, missing_pred])
if n_missing > 0:
for i in range(n_missing):
image_performance.append(0)
output = dict()
for i, performance in enumerate(image_performance):
if i > 0:
output["instrument_{}".format(i - 1)] = performance
else:
output["background"] = performance
return output
diff --git a/evaluation/distance_calculations.py b/statistics/distance_calculations.py
similarity index 90%
rename from evaluation/distance_calculations.py
rename to statistics/distance_calculations.py
index 1545566..a2121b7 100644
--- a/evaluation/distance_calculations.py
+++ b/statistics/distance_calculations.py
@@ -1,475 +1,544 @@
import numpy as np
import scipy.ndimage
+from scipy.optimize import linear_sum_assignment as hungarian_algorithm
# neighbour_code_to_normals is a lookup table.
# For every binary neighbour code
# (2x2x2 neighbourhood = 8 neighbours = 8 bits = 256 codes)
# it contains the surface normals of the triangles (called "surfel" for
# "surface element" in the following). The length of the normal
# vector encodes the surfel area.
#
# created by compute_surface_area_lookup_table.ipynb using the
# marching_cube algorithm, see e.g. https://en.wikipedia.org/wiki/Marching_cubes
#
neighbour_code_to_normals = [
[[0,0,0]],
[[0.125,0.125,0.125]],
[[-0.125,-0.125,0.125]],
[[-0.25,-0.25,0.0],[0.25,0.25,-0.0]],
[[0.125,-0.125,0.125]],
[[-0.25,-0.0,-0.25],[0.25,0.0,0.25]],
[[0.125,-0.125,0.125],[-0.125,-0.125,0.125]],
[[0.5,0.0,-0.0],[0.25,0.25,0.25],[0.125,0.125,0.125]],
[[-0.125,0.125,0.125]],
[[0.125,0.125,0.125],[-0.125,0.125,0.125]],
[[-0.25,0.0,0.25],[-0.25,0.0,0.25]],
[[0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]],
[[0.25,-0.25,0.0],[0.25,-0.25,0.0]],
[[0.5,0.0,0.0],[0.25,-0.25,0.25],[-0.125,0.125,-0.125]],
[[-0.5,0.0,0.0],[-0.25,0.25,0.25],[-0.125,0.125,0.125]],
[[0.5,0.0,0.0],[0.5,0.0,0.0]],
[[0.125,-0.125,-0.125]],
[[0.0,-0.25,-0.25],[0.0,0.25,0.25]],
[[-0.125,-0.125,0.125],[0.125,-0.125,-0.125]],
[[0.0,-0.5,0.0],[0.25,0.25,0.25],[0.125,0.125,0.125]],
[[0.125,-0.125,0.125],[0.125,-0.125,-0.125]],
[[0.0,0.0,-0.5],[0.25,0.25,0.25],[-0.125,-0.125,-0.125]],
[[-0.125,-0.125,0.125],[0.125,-0.125,0.125],[0.125,-0.125,-0.125]],
[[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25],[0.25,0.25,0.25],[0.125,0.125,0.125]],
[[-0.125,0.125,0.125],[0.125,-0.125,-0.125]],
[[0.0,-0.25,-0.25],[0.0,0.25,0.25],[-0.125,0.125,0.125]],
[[-0.25,0.0,0.25],[-0.25,0.0,0.25],[0.125,-0.125,-0.125]],
[[0.125,0.125,0.125],[0.375,0.375,0.375],[0.0,-0.25,0.25],[-0.25,0.0,0.25]],
[[0.125,-0.125,-0.125],[0.25,-0.25,0.0],[0.25,-0.25,0.0]],
[[0.375,0.375,0.375],[0.0,0.25,-0.25],[-0.125,-0.125,-0.125],[-0.25,0.25,0.0]],
[[-0.5,0.0,0.0],[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25],[0.125,0.125,0.125]],
[[-0.5,0.0,0.0],[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25]],
[[0.125,-0.125,0.125]],
[[0.125,0.125,0.125],[0.125,-0.125,0.125]],
[[0.0,-0.25,0.25],[0.0,0.25,-0.25]],
[[0.0,-0.5,0.0],[0.125,0.125,-0.125],[0.25,0.25,-0.25]],
[[0.125,-0.125,0.125],[0.125,-0.125,0.125]],
[[0.125,-0.125,0.125],[-0.25,-0.0,-0.25],[0.25,0.0,0.25]],
[[0.0,-0.25,0.25],[0.0,0.25,-0.25],[0.125,-0.125,0.125]],
[[-0.375,-0.375,0.375],[-0.0,0.25,0.25],[0.125,0.125,-0.125],[-0.25,-0.0,-0.25]],
[[-0.125,0.125,0.125],[0.125,-0.125,0.125]],
[[0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,0.125,0.125]],
[[-0.0,0.0,0.5],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]],
[[0.25,0.25,-0.25],[0.25,0.25,-0.25],[0.125,0.125,-0.125],[-0.125,-0.125,0.125]],
[[0.125,-0.125,0.125],[0.25,-0.25,0.0],[0.25,-0.25,0.0]],
[[0.5,0.0,0.0],[0.25,-0.25,0.25],[-0.125,0.125,-0.125],[0.125,-0.125,0.125]],
[[0.0,0.25,-0.25],[0.375,-0.375,-0.375],[-0.125,0.125,0.125],[0.25,0.25,0.0]],
[[-0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]],
[[0.25,-0.25,0.0],[-0.25,0.25,0.0]],
[[0.0,0.5,0.0],[-0.25,0.25,0.25],[0.125,-0.125,-0.125]],
[[0.0,0.5,0.0],[0.125,-0.125,0.125],[-0.25,0.25,-0.25]],
[[0.0,0.5,0.0],[0.0,-0.5,0.0]],
[[0.25,-0.25,0.0],[-0.25,0.25,0.0],[0.125,-0.125,0.125]],
[[-0.375,-0.375,-0.375],[-0.25,0.0,0.25],[-0.125,-0.125,-0.125],[-0.25,0.25,0.0]],
[[0.125,0.125,0.125],[0.0,-0.5,0.0],[-0.25,-0.25,-0.25],[-0.125,-0.125,-0.125]],
[[0.0,-0.5,0.0],[-0.25,-0.25,-0.25],[-0.125,-0.125,-0.125]],
[[-0.125,0.125,0.125],[0.25,-0.25,0.0],[-0.25,0.25,0.0]],
[[0.0,0.5,0.0],[0.25,0.25,-0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]],
[[-0.375,0.375,-0.375],[-0.25,-0.25,0.0],[-0.125,0.125,-0.125],[-0.25,0.0,0.25]],
[[0.0,0.5,0.0],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]],
[[0.25,-0.25,0.0],[-0.25,0.25,0.0],[0.25,-0.25,0.0],[0.25,-0.25,0.0]],
[[-0.25,-0.25,0.0],[-0.25,-0.25,0.0],[-0.125,-0.125,0.125]],
[[0.125,0.125,0.125],[-0.25,-0.25,0.0],[-0.25,-0.25,0.0]],
[[-0.25,-0.25,0.0],[-0.25,-0.25,0.0]],
[[-0.125,-0.125,0.125]],
[[0.125,0.125,0.125],[-0.125,-0.125,0.125]],
[[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]],
[[-0.125,-0.125,0.125],[-0.25,-0.25,0.0],[0.25,0.25,-0.0]],
[[0.0,-0.25,0.25],[0.0,-0.25,0.25]],
[[0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125]],
[[0.0,-0.25,0.25],[0.0,-0.25,0.25],[-0.125,-0.125,0.125]],
[[0.375,-0.375,0.375],[0.0,-0.25,-0.25],[-0.125,0.125,-0.125],[0.25,0.25,0.0]],
[[-0.125,-0.125,0.125],[-0.125,0.125,0.125]],
[[0.125,0.125,0.125],[-0.125,-0.125,0.125],[-0.125,0.125,0.125]],
[[-0.125,-0.125,0.125],[-0.25,0.0,0.25],[-0.25,0.0,0.25]],
[[0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]],
[[-0.0,0.5,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125]],
[[-0.25,0.25,-0.25],[-0.25,0.25,-0.25],[-0.125,0.125,-0.125],[-0.125,0.125,-0.125]],
[[-0.25,0.0,-0.25],[0.375,-0.375,-0.375],[0.0,0.25,-0.25],[-0.125,0.125,0.125]],
[[0.5,0.0,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125]],
[[-0.25,0.0,0.25],[0.25,0.0,-0.25]],
[[-0.0,0.0,0.5],[-0.25,0.25,0.25],[-0.125,0.125,0.125]],
[[-0.125,-0.125,0.125],[-0.25,0.0,0.25],[0.25,0.0,-0.25]],
[[-0.25,-0.0,-0.25],[-0.375,0.375,0.375],[-0.25,-0.25,0.0],[-0.125,0.125,0.125]],
[[0.0,0.0,-0.5],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]],
[[-0.0,0.0,0.5],[0.0,0.0,0.5]],
[[0.125,0.125,0.125],[0.125,0.125,0.125],[0.25,0.25,0.25],[0.0,0.0,0.5]],
[[0.125,0.125,0.125],[0.25,0.25,0.25],[0.0,0.0,0.5]],
[[-0.25,0.0,0.25],[0.25,0.0,-0.25],[-0.125,0.125,0.125]],
[[-0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]],
[[-0.25,0.0,0.25],[-0.25,0.0,0.25],[-0.25,0.0,0.25],[0.25,0.0,-0.25]],
[[0.125,-0.125,0.125],[0.25,0.0,0.25],[0.25,0.0,0.25]],
[[0.25,0.0,0.25],[-0.375,-0.375,0.375],[-0.25,0.25,0.0],[-0.125,-0.125,0.125]],
[[-0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125]],
[[0.125,0.125,0.125],[0.25,0.0,0.25],[0.25,0.0,0.25]],
[[0.25,0.0,0.25],[0.25,0.0,0.25]],
[[-0.125,-0.125,0.125],[0.125,-0.125,0.125]],
[[0.125,0.125,0.125],[-0.125,-0.125,0.125],[0.125,-0.125,0.125]],
[[-0.125,-0.125,0.125],[0.0,-0.25,0.25],[0.0,0.25,-0.25]],
[[0.0,-0.5,0.0],[0.125,0.125,-0.125],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]],
[[0.0,-0.25,0.25],[0.0,-0.25,0.25],[0.125,-0.125,0.125]],
[[0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]],
[[0.0,-0.25,0.25],[0.0,-0.25,0.25],[0.0,-0.25,0.25],[0.0,0.25,-0.25]],
[[0.0,0.25,0.25],[0.0,0.25,0.25],[0.125,-0.125,-0.125]],
[[-0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,-0.125,0.125]],
[[-0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,-0.125,0.125],[0.125,0.125,0.125]],
[[-0.0,0.0,0.5],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]],
[[0.125,0.125,0.125],[0.125,-0.125,0.125],[0.125,-0.125,-0.125]],
[[-0.0,0.5,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]],
[[0.125,0.125,0.125],[-0.125,-0.125,0.125],[0.125,-0.125,-0.125]],
[[0.0,-0.25,-0.25],[0.0,0.25,0.25],[0.125,0.125,0.125]],
[[0.125,0.125,0.125],[0.125,-0.125,-0.125]],
[[0.5,0.0,-0.0],[0.25,-0.25,-0.25],[0.125,-0.125,-0.125]],
[[-0.25,0.25,0.25],[-0.125,0.125,0.125],[-0.25,0.25,0.25],[0.125,-0.125,-0.125]],
[[0.375,-0.375,0.375],[0.0,0.25,0.25],[-0.125,0.125,-0.125],[-0.25,0.0,0.25]],
[[0.0,-0.5,0.0],[-0.25,0.25,0.25],[-0.125,0.125,0.125]],
[[-0.375,-0.375,0.375],[0.25,-0.25,0.0],[0.0,0.25,0.25],[-0.125,-0.125,0.125]],
[[-0.125,0.125,0.125],[-0.25,0.25,0.25],[0.0,0.0,0.5]],
[[0.125,0.125,0.125],[0.0,0.25,0.25],[0.0,0.25,0.25]],
[[0.0,0.25,0.25],[0.0,0.25,0.25]],
[[0.5,0.0,-0.0],[0.25,0.25,0.25],[0.125,0.125,0.125],[0.125,0.125,0.125]],
[[0.125,-0.125,0.125],[-0.125,-0.125,0.125],[0.125,0.125,0.125]],
[[-0.25,-0.0,-0.25],[0.25,0.0,0.25],[0.125,0.125,0.125]],
[[0.125,0.125,0.125],[0.125,-0.125,0.125]],
[[-0.25,-0.25,0.0],[0.25,0.25,-0.0],[0.125,0.125,0.125]],
[[0.125,0.125,0.125],[-0.125,-0.125,0.125]],
[[0.125,0.125,0.125],[0.125,0.125,0.125]],
[[0.125,0.125,0.125]],
[[0.125,0.125,0.125]],
[[0.125,0.125,0.125],[0.125,0.125,0.125]],
[[0.125,0.125,0.125],[-0.125,-0.125,0.125]],
[[-0.25,-0.25,0.0],[0.25,0.25,-0.0],[0.125,0.125,0.125]],
[[0.125,0.125,0.125],[0.125,-0.125,0.125]],
[[-0.25,-0.0,-0.25],[0.25,0.0,0.25],[0.125,0.125,0.125]],
[[0.125,-0.125,0.125],[-0.125,-0.125,0.125],[0.125,0.125,0.125]],
[[0.5,0.0,-0.0],[0.25,0.25,0.25],[0.125,0.125,0.125],[0.125,0.125,0.125]],
[[0.0,0.25,0.25],[0.0,0.25,0.25]],
[[0.125,0.125,0.125],[0.0,0.25,0.25],[0.0,0.25,0.25]],
[[-0.125,0.125,0.125],[-0.25,0.25,0.25],[0.0,0.0,0.5]],
[[-0.375,-0.375,0.375],[0.25,-0.25,0.0],[0.0,0.25,0.25],[-0.125,-0.125,0.125]],
[[0.0,-0.5,0.0],[-0.25,0.25,0.25],[-0.125,0.125,0.125]],
[[0.375,-0.375,0.375],[0.0,0.25,0.25],[-0.125,0.125,-0.125],[-0.25,0.0,0.25]],
[[-0.25,0.25,0.25],[-0.125,0.125,0.125],[-0.25,0.25,0.25],[0.125,-0.125,-0.125]],
[[0.5,0.0,-0.0],[0.25,-0.25,-0.25],[0.125,-0.125,-0.125]],
[[0.125,0.125,0.125],[0.125,-0.125,-0.125]],
[[0.0,-0.25,-0.25],[0.0,0.25,0.25],[0.125,0.125,0.125]],
[[0.125,0.125,0.125],[-0.125,-0.125,0.125],[0.125,-0.125,-0.125]],
[[-0.0,0.5,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]],
[[0.125,0.125,0.125],[0.125,-0.125,0.125],[0.125,-0.125,-0.125]],
[[-0.0,0.0,0.5],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]],
[[-0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,-0.125,0.125],[0.125,0.125,0.125]],
[[-0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,-0.125,0.125]],
[[0.0,0.25,0.25],[0.0,0.25,0.25],[0.125,-0.125,-0.125]],
[[0.0,-0.25,-0.25],[0.0,0.25,0.25],[0.0,0.25,0.25],[0.0,0.25,0.25]],
[[0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]],
[[0.0,-0.25,0.25],[0.0,-0.25,0.25],[0.125,-0.125,0.125]],
[[0.0,-0.5,0.0],[0.125,0.125,-0.125],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]],
[[-0.125,-0.125,0.125],[0.0,-0.25,0.25],[0.0,0.25,-0.25]],
[[0.125,0.125,0.125],[-0.125,-0.125,0.125],[0.125,-0.125,0.125]],
[[-0.125,-0.125,0.125],[0.125,-0.125,0.125]],
[[0.25,0.0,0.25],[0.25,0.0,0.25]],
[[0.125,0.125,0.125],[0.25,0.0,0.25],[0.25,0.0,0.25]],
[[-0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125]],
[[0.25,0.0,0.25],[-0.375,-0.375,0.375],[-0.25,0.25,0.0],[-0.125,-0.125,0.125]],
[[0.125,-0.125,0.125],[0.25,0.0,0.25],[0.25,0.0,0.25]],
[[-0.25,-0.0,-0.25],[0.25,0.0,0.25],[0.25,0.0,0.25],[0.25,0.0,0.25]],
[[-0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]],
[[-0.25,0.0,0.25],[0.25,0.0,-0.25],[-0.125,0.125,0.125]],
[[0.125,0.125,0.125],[0.25,0.25,0.25],[0.0,0.0,0.5]],
[[0.125,0.125,0.125],[0.125,0.125,0.125],[0.25,0.25,0.25],[0.0,0.0,0.5]],
[[-0.0,0.0,0.5],[0.0,0.0,0.5]],
[[0.0,0.0,-0.5],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]],
[[-0.25,-0.0,-0.25],[-0.375,0.375,0.375],[-0.25,-0.25,0.0],[-0.125,0.125,0.125]],
[[-0.125,-0.125,0.125],[-0.25,0.0,0.25],[0.25,0.0,-0.25]],
[[-0.0,0.0,0.5],[-0.25,0.25,0.25],[-0.125,0.125,0.125]],
[[-0.25,0.0,0.25],[0.25,0.0,-0.25]],
[[0.5,0.0,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125]],
[[-0.25,0.0,-0.25],[0.375,-0.375,-0.375],[0.0,0.25,-0.25],[-0.125,0.125,0.125]],
[[-0.25,0.25,-0.25],[-0.25,0.25,-0.25],[-0.125,0.125,-0.125],[-0.125,0.125,-0.125]],
[[-0.0,0.5,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125]],
[[0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]],
[[-0.125,-0.125,0.125],[-0.25,0.0,0.25],[-0.25,0.0,0.25]],
[[0.125,0.125,0.125],[-0.125,-0.125,0.125],[-0.125,0.125,0.125]],
[[-0.125,-0.125,0.125],[-0.125,0.125,0.125]],
[[0.375,-0.375,0.375],[0.0,-0.25,-0.25],[-0.125,0.125,-0.125],[0.25,0.25,0.0]],
[[0.0,-0.25,0.25],[0.0,-0.25,0.25],[-0.125,-0.125,0.125]],
[[0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125]],
[[0.0,-0.25,0.25],[0.0,-0.25,0.25]],
[[-0.125,-0.125,0.125],[-0.25,-0.25,0.0],[0.25,0.25,-0.0]],
[[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]],
[[0.125,0.125,0.125],[-0.125,-0.125,0.125]],
[[-0.125,-0.125,0.125]],
[[-0.25,-0.25,0.0],[-0.25,-0.25,0.0]],
[[0.125,0.125,0.125],[-0.25,-0.25,0.0],[-0.25,-0.25,0.0]],
[[-0.25,-0.25,0.0],[-0.25,-0.25,0.0],[-0.125,-0.125,0.125]],
[[-0.25,-0.25,0.0],[-0.25,-0.25,0.0],[-0.25,-0.25,0.0],[0.25,0.25,-0.0]],
[[0.0,0.5,0.0],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]],
[[-0.375,0.375,-0.375],[-0.25,-0.25,0.0],[-0.125,0.125,-0.125],[-0.25,0.0,0.25]],
[[0.0,0.5,0.0],[0.25,0.25,-0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]],
[[-0.125,0.125,0.125],[0.25,-0.25,0.0],[-0.25,0.25,0.0]],
[[0.0,-0.5,0.0],[-0.25,-0.25,-0.25],[-0.125,-0.125,-0.125]],
[[0.125,0.125,0.125],[0.0,-0.5,0.0],[-0.25,-0.25,-0.25],[-0.125,-0.125,-0.125]],
[[-0.375,-0.375,-0.375],[-0.25,0.0,0.25],[-0.125,-0.125,-0.125],[-0.25,0.25,0.0]],
[[0.25,-0.25,0.0],[-0.25,0.25,0.0],[0.125,-0.125,0.125]],
[[0.0,0.5,0.0],[0.0,-0.5,0.0]],
[[0.0,0.5,0.0],[0.125,-0.125,0.125],[-0.25,0.25,-0.25]],
[[0.0,0.5,0.0],[-0.25,0.25,0.25],[0.125,-0.125,-0.125]],
[[0.25,-0.25,0.0],[-0.25,0.25,0.0]],
[[-0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]],
[[0.0,0.25,-0.25],[0.375,-0.375,-0.375],[-0.125,0.125,0.125],[0.25,0.25,0.0]],
[[0.5,0.0,0.0],[0.25,-0.25,0.25],[-0.125,0.125,-0.125],[0.125,-0.125,0.125]],
[[0.125,-0.125,0.125],[0.25,-0.25,0.0],[0.25,-0.25,0.0]],
[[0.25,0.25,-0.25],[0.25,0.25,-0.25],[0.125,0.125,-0.125],[-0.125,-0.125,0.125]],
[[-0.0,0.0,0.5],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]],
[[0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,0.125,0.125]],
[[-0.125,0.125,0.125],[0.125,-0.125,0.125]],
[[-0.375,-0.375,0.375],[-0.0,0.25,0.25],[0.125,0.125,-0.125],[-0.25,-0.0,-0.25]],
[[0.0,-0.25,0.25],[0.0,0.25,-0.25],[0.125,-0.125,0.125]],
[[0.125,-0.125,0.125],[-0.25,-0.0,-0.25],[0.25,0.0,0.25]],
[[0.125,-0.125,0.125],[0.125,-0.125,0.125]],
[[0.0,-0.5,0.0],[0.125,0.125,-0.125],[0.25,0.25,-0.25]],
[[0.0,-0.25,0.25],[0.0,0.25,-0.25]],
[[0.125,0.125,0.125],[0.125,-0.125,0.125]],
[[0.125,-0.125,0.125]],
[[-0.5,0.0,0.0],[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25]],
[[-0.5,0.0,0.0],[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25],[0.125,0.125,0.125]],
[[0.375,0.375,0.375],[0.0,0.25,-0.25],[-0.125,-0.125,-0.125],[-0.25,0.25,0.0]],
[[0.125,-0.125,-0.125],[0.25,-0.25,0.0],[0.25,-0.25,0.0]],
[[0.125,0.125,0.125],[0.375,0.375,0.375],[0.0,-0.25,0.25],[-0.25,0.0,0.25]],
[[-0.25,0.0,0.25],[-0.25,0.0,0.25],[0.125,-0.125,-0.125]],
[[0.0,-0.25,-0.25],[0.0,0.25,0.25],[-0.125,0.125,0.125]],
[[-0.125,0.125,0.125],[0.125,-0.125,-0.125]],
[[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25],[0.25,0.25,0.25],[0.125,0.125,0.125]],
[[-0.125,-0.125,0.125],[0.125,-0.125,0.125],[0.125,-0.125,-0.125]],
[[0.0,0.0,-0.5],[0.25,0.25,0.25],[-0.125,-0.125,-0.125]],
[[0.125,-0.125,0.125],[0.125,-0.125,-0.125]],
[[0.0,-0.5,0.0],[0.25,0.25,0.25],[0.125,0.125,0.125]],
[[-0.125,-0.125,0.125],[0.125,-0.125,-0.125]],
[[0.0,-0.25,-0.25],[0.0,0.25,0.25]],
[[0.125,-0.125,-0.125]],
[[0.5,0.0,0.0],[0.5,0.0,0.0]],
[[-0.5,0.0,0.0],[-0.25,0.25,0.25],[-0.125,0.125,0.125]],
[[0.5,0.0,0.0],[0.25,-0.25,0.25],[-0.125,0.125,-0.125]],
[[0.25,-0.25,0.0],[0.25,-0.25,0.0]],
[[0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]],
[[-0.25,0.0,0.25],[-0.25,0.0,0.25]],
[[0.125,0.125,0.125],[-0.125,0.125,0.125]],
[[-0.125,0.125,0.125]],
[[0.5,0.0,-0.0],[0.25,0.25,0.25],[0.125,0.125,0.125]],
[[0.125,-0.125,0.125],[-0.125,-0.125,0.125]],
[[-0.25,-0.0,-0.25],[0.25,0.0,0.25]],
[[0.125,-0.125,0.125]],
[[-0.25,-0.25,0.0],[0.25,0.25,-0.0]],
[[-0.125,-0.125,0.125]],
[[0.125,0.125,0.125]],
[[0,0,0]]]
def compute_surface_distances(mask_gt, mask_pred, spacing_mm):
"""Compute closest distances from all surface points to the other surface.
Finds all surface elements "surfels" in the ground truth mask `mask_gt` and
the predicted mask `mask_pred`, computes their area in mm^2 and the distance
to the closest point on the other surface. It returns two sorted lists of
distances together with the corresponding surfel areas. If one of the masks
is empty, the corresponding lists are empty and all distances in the other
list are `inf`
Args:
mask_gt: 3-dim Numpy array of type bool. The ground truth mask.
mask_pred: 3-dim Numpy array of type bool. The predicted mask.
spacing_mm: 3-element list-like structure. Voxel spacing in x0, x1 and x2
direction
Returns:
A dict with
"distances_gt_to_pred": 1-dim numpy array of type float. The distances in mm
from all ground truth surface elements to the predicted surface,
sorted from smallest to largest
"distances_pred_to_gt": 1-dim numpy array of type float. The distances in mm
from all predicted surface elements to the ground truth surface,
sorted from smallest to largest
"surfel_areas_gt": 1-dim numpy array of type float. The area in mm^2 of
the ground truth surface elements in the same order as
distances_gt_to_pred
"surfel_areas_pred": 1-dim numpy array of type float. The area in mm^2 of
the predicted surface elements in the same order as
distances_pred_to_gt
"""
# compute the area for all 256 possible surface elements
# (given a 2x2x2 neighbourhood) according to the spacing_mm
neighbour_code_to_surface_area = np.zeros([256])
for code in range(256):
normals = np.array(neighbour_code_to_normals[code])
sum_area = 0
for normal_idx in range(normals.shape[0]):
# normal vector
n = np.zeros([3])
n[0] = normals[normal_idx, 0] * spacing_mm[1] * spacing_mm[2]
n[1] = normals[normal_idx, 1] * spacing_mm[0] * spacing_mm[2]
n[2] = normals[normal_idx, 2] * spacing_mm[0] * spacing_mm[1]
area = np.linalg.norm(n)
sum_area += area
neighbour_code_to_surface_area[code] = sum_area
# compute the bounding box of the masks to trim
# the volume to the smallest possible processing subvolume
mask_all = mask_gt | mask_pred
bbox_min = np.zeros(3, np.int64)
bbox_max = np.zeros(3, np.int64)
# max projection to the x0-axis
proj_0 = np.max(np.max(mask_all, axis=2), axis=1)
idx_nonzero_0 = np.nonzero(proj_0)[0]
if len(idx_nonzero_0) == 0:
return {"distances_gt_to_pred": np.array([]),
"distances_pred_to_gt": np.array([]),
"surfel_areas_gt": np.array([]),
"surfel_areas_pred": np.array([])}
bbox_min[0] = np.min(idx_nonzero_0)
bbox_max[0] = np.max(idx_nonzero_0)
# max projection to the x1-axis
proj_1 = np.max(np.max(mask_all, axis=2), axis=0)
idx_nonzero_1 = np.nonzero(proj_1)[0]
bbox_min[1] = np.min(idx_nonzero_1)
bbox_max[1] = np.max(idx_nonzero_1)
# max projection to the x2-axis
proj_2 = np.max(np.max(mask_all, axis=1), axis=0)
idx_nonzero_2 = np.nonzero(proj_2)[0]
bbox_min[2] = np.min(idx_nonzero_2)
bbox_max[2] = np.max(idx_nonzero_2)
- print("bounding box min = {}".format(bbox_min))
- print("bounding box max = {}".format(bbox_max))
+ # print("bounding box min = {}".format(bbox_min))
+ # print("bounding box max = {}".format(bbox_max))
# crop the processing subvolume.
# we need to zeropad the cropped region with 1 voxel at the lower,
# the right and the back side. This is required to obtain the "full"
# convolution result with the 2x2x2 kernel
cropmask_gt = np.zeros((bbox_max - bbox_min) + 2, np.uint8)
cropmask_pred = np.zeros((bbox_max - bbox_min) + 2, np.uint8)
cropmask_gt[0:-1, 0:-1, 0:-1] = mask_gt[bbox_min[0]:bbox_max[0] + 1,
bbox_min[1]:bbox_max[1] + 1,
bbox_min[2]:bbox_max[2] + 1]
cropmask_pred[0:-1, 0:-1, 0:-1] = mask_pred[bbox_min[0]:bbox_max[0] + 1,
bbox_min[1]:bbox_max[1] + 1,
bbox_min[2]:bbox_max[2] + 1]
# compute the neighbour code (local binary pattern) for each voxel
# the resultsing arrays are spacially shifted by minus half a voxel in each axis.
# i.e. the points are located at the corners of the original voxels
kernel = np.array([[[128, 64],
[32, 16]],
[[8, 4],
[2, 1]]])
neighbour_code_map_gt = scipy.ndimage.filters.correlate(cropmask_gt.astype(np.uint8), kernel, mode="constant",
cval=0)
neighbour_code_map_pred = scipy.ndimage.filters.correlate(cropmask_pred.astype(np.uint8), kernel, mode="constant",
cval=0)
# create masks with the surface voxels
borders_gt = ((neighbour_code_map_gt != 0) & (neighbour_code_map_gt != 255))
borders_pred = ((neighbour_code_map_pred != 0) & (neighbour_code_map_pred != 255))
# compute the distance transform (closest distance of each voxel to the surface voxels)
if borders_gt.any():
distmap_gt = scipy.ndimage.morphology.distance_transform_edt(~borders_gt, sampling=spacing_mm)
else:
distmap_gt = np.Inf * np.ones(borders_gt.shape)
if borders_pred.any():
distmap_pred = scipy.ndimage.morphology.distance_transform_edt(~borders_pred, sampling=spacing_mm)
else:
distmap_pred = np.Inf * np.ones(borders_pred.shape)
# compute the area of each surface element
surface_area_map_gt = neighbour_code_to_surface_area[neighbour_code_map_gt]
surface_area_map_pred = neighbour_code_to_surface_area[neighbour_code_map_pred]
# create a list of all surface elements with distance and area
distances_gt_to_pred = distmap_pred[borders_gt]
distances_pred_to_gt = distmap_gt[borders_pred]
surfel_areas_gt = surface_area_map_gt[borders_gt]
surfel_areas_pred = surface_area_map_pred[borders_pred]
# sort them by distance
if distances_gt_to_pred.shape != (0,):
sorted_surfels_gt = np.array(sorted(zip(distances_gt_to_pred, surfel_areas_gt)))
distances_gt_to_pred = sorted_surfels_gt[:, 0]
surfel_areas_gt = sorted_surfels_gt[:, 1]
if distances_pred_to_gt.shape != (0,):
sorted_surfels_pred = np.array(sorted(zip(distances_pred_to_gt, surfel_areas_pred)))
distances_pred_to_gt = sorted_surfels_pred[:, 0]
surfel_areas_pred = sorted_surfels_pred[:, 1]
return {"distances_gt_to_pred": distances_gt_to_pred,
"distances_pred_to_gt": distances_pred_to_gt,
"surfel_areas_gt": surfel_areas_gt,
"surfel_areas_pred": surfel_areas_pred}
def compute_average_surface_distance(surface_distances):
distances_gt_to_pred = surface_distances["distances_gt_to_pred"]
distances_pred_to_gt = surface_distances["distances_pred_to_gt"]
surfel_areas_gt = surface_distances["surfel_areas_gt"]
surfel_areas_pred = surface_distances["surfel_areas_pred"]
average_distance_gt_to_pred = np.sum(distances_gt_to_pred * surfel_areas_gt) / np.sum(surfel_areas_gt)
average_distance_pred_to_gt = np.sum(distances_pred_to_gt * surfel_areas_pred) / np.sum(surfel_areas_pred)
return (average_distance_gt_to_pred, average_distance_pred_to_gt)
def compute_robust_hausdorff(surface_distances, percent):
distances_gt_to_pred = surface_distances["distances_gt_to_pred"]
distances_pred_to_gt = surface_distances["distances_pred_to_gt"]
surfel_areas_gt = surface_distances["surfel_areas_gt"]
surfel_areas_pred = surface_distances["surfel_areas_pred"]
if len(distances_gt_to_pred) > 0:
surfel_areas_cum_gt = np.cumsum(surfel_areas_gt) / np.sum(surfel_areas_gt)
idx = np.searchsorted(surfel_areas_cum_gt, percent / 100.0)
perc_distance_gt_to_pred = distances_gt_to_pred[min(idx, len(distances_gt_to_pred) - 1)]
else:
perc_distance_gt_to_pred = np.Inf
if len(distances_pred_to_gt) > 0:
surfel_areas_cum_pred = np.cumsum(surfel_areas_pred) / np.sum(surfel_areas_pred)
idx = np.searchsorted(surfel_areas_cum_pred, percent / 100.0)
perc_distance_pred_to_gt = distances_pred_to_gt[min(idx, len(distances_pred_to_gt) - 1)]
else:
perc_distance_pred_to_gt = np.Inf
return max(perc_distance_gt_to_pred, perc_distance_pred_to_gt)
def compute_surface_overlap_at_tolerance(surface_distances, tolerance_mm):
distances_gt_to_pred = surface_distances["distances_gt_to_pred"]
distances_pred_to_gt = surface_distances["distances_pred_to_gt"]
surfel_areas_gt = surface_distances["surfel_areas_gt"]
surfel_areas_pred = surface_distances["surfel_areas_pred"]
rel_overlap_gt = np.sum(surfel_areas_gt[distances_gt_to_pred <= tolerance_mm]) / np.sum(surfel_areas_gt)
rel_overlap_pred = np.sum(surfel_areas_pred[distances_pred_to_gt <= tolerance_mm]) / np.sum(surfel_areas_pred)
return (rel_overlap_gt, rel_overlap_pred)
def compute_surface_dice_at_tolerance(surface_distances, tolerance_mm):
distances_gt_to_pred = surface_distances["distances_gt_to_pred"]
distances_pred_to_gt = surface_distances["distances_pred_to_gt"]
surfel_areas_gt = surface_distances["surfel_areas_gt"]
surfel_areas_pred = surface_distances["surfel_areas_pred"]
overlap_gt = np.sum(surfel_areas_gt[distances_gt_to_pred <= tolerance_mm])
overlap_pred = np.sum(surfel_areas_pred[distances_pred_to_gt <= tolerance_mm])
surface_dice = (overlap_gt + overlap_pred) / (
np.sum(surfel_areas_gt) + np.sum(surfel_areas_pred))
- return surface_dice
\ No newline at end of file
+ return surface_dice
+
+
+def compute_multi_instance_surface_dice_at_tolerance(mask_gt, mask_pred, spacing_mm=[1, 1, 1], tolerance=13):
+ # get number of labels in image
+ instances_gt = np.unique(mask_gt)
+ instances_pred = np.unique(mask_pred)
+
+ # create performance matrix
+ performance_matrix = np.zeros((len(instances_gt), len(instances_pred)))
+ masks = []
+
+ # calculate dice score for each ground truth to predicted instance
+ for counter_gt, instance_gt in enumerate(instances_gt):
+
+ # create binary mask for current gt instance
+ gt = mask_gt.copy()
+ gt[mask_gt != instance_gt] = 0
+ gt[mask_gt == instance_gt] = 1
+
+ masks_row = []
+ for counter_pred, instance_pred in enumerate(instances_pred):
+ # make binary mask for current predicted instance
+ prediction = mask_pred.copy()
+ prediction[mask_pred != instance_pred] = 0
+ prediction[mask_pred == instance_pred] = 1
+
+ # add additional dimension
+ if prediction.ndim < 3:
+ prediction = np.expand_dims(prediction, axis=2)
+ if gt.ndim < 3:
+ gt = np.expand_dims(gt, axis=2)
+
+ # calculate surfaces
+ surface_distances = compute_surface_distances(mask_gt=gt,
+ mask_pred=prediction,
+ spacing_mm=spacing_mm)
+
+ # calculate dice
+ performance_matrix[counter_gt, counter_pred] = compute_surface_dice_at_tolerance(surface_distances, tolerance)
+ masks_row.append([gt, prediction])
+ masks.append(masks_row)
+
+ # assign instrument instances according to hungarian algorithm
+ label_assignment = hungarian_algorithm(performance_matrix * -1)
+ label_nr_gt, label_nr_pred = label_assignment
+
+ # get performance per instance
+ image_performance = []
+ for i in range(len(label_nr_gt)):
+ instance_dice = performance_matrix[label_nr_gt[i], label_nr_pred[i]]
+ image_performance.append(instance_dice)
+
+ missing_pred = np.absolute(len(instances_pred) - len(image_performance))
+ missing_gt = np.absolute(len(instances_gt) - len(image_performance))
+ n_missing = np.max([missing_gt, missing_pred])
+
+ if n_missing > 0:
+ for i in range(n_missing):
+ image_performance.append(0)
+
+ output = dict()
+ for i, performance in enumerate(image_performance):
+ if i > 0:
+ output["instrument_{}".format(i - 1)] = performance
+ else:
+ output["background"] = performance
+
+ return output
\ No newline at end of file
diff --git a/evaluation/mean_average_precision_calculations.py b/statistics/mean_average_precision_calculations.py
similarity index 79%
rename from evaluation/mean_average_precision_calculations.py
rename to statistics/mean_average_precision_calculations.py
index 5de1cd2..c144dd5 100644
--- a/evaluation/mean_average_precision_calculations.py
+++ b/statistics/mean_average_precision_calculations.py
@@ -1,183 +1,200 @@
+import time
+
import numpy as np
from pandas import DataFrame
from scipy.optimize import linear_sum_assignment as hungarian_algorithm
+import matplotlib.pyplot as plt
def compute_iou(mask_gt, mask_pred):
"""
Compute the intersection over union (https://en.wikipedia.org/wiki/Jaccard_index)
compute the intersectin over union between the ground truth mask `mask_gt`
and the predicted mask `mask_pred`.
Args:
mask_gt: 3-dim Numpy array of type bool. The ground truth mask.
mask_pred: 3-dim Numpy array of type bool. The predicted mask.
Returns:
the iou coeffcient as float. If both masks are empty, the result is 0
"""
mask_gt = mask_gt.astype('bool')
mask_pred = mask_pred.astype('bool')
overlap = mask_gt * mask_pred # Logical AND
union = mask_gt + mask_pred # Logical OR
iou = overlap.sum() / float(union.sum()) # Treats "True" as 1,
return iou
-def compute_statistics(mask_gt, mask_pred):
+def compute_statistics(mask_gt, mask_pred, min_iou_for_match=0.03):
"""
Compute Statistic
compute statistics (TP, FP, FN, precision, recall) between the ground truth mask `mask_gt`
and the predicted mask `mask_pred`.
TP = True positive (defined as an iou>=0.03)
FP = False positive
FN = False negative
precision = true_positive / (true_positive + false_positive)
recall = true_positive / (true_positive + false_negative)
Args:
mask_gt: 3-dim Numpy array of type bool. The ground truth mask.
mask_pred: 3-dim Numpy array of type bool. The predicted mask.
Returns:
output = dict(
true_positive=true_positive,
false_positive=false_positive,
false_negative=false_negative,
precision=precision,
recall=recall
)
"""
- # define constants
- min_iou_for_match = 0.03
# get number of labels in image
instances_gt = list(np.unique(mask_gt))
instances_pred = list(np.unique(mask_pred))
# remove background
instances_gt = instances_gt[1:]
instances_pred = instances_pred[1:]
# create performance matrix
performance_matrix = np.zeros((len(instances_gt), len(instances_pred)))
masks = []
# calculate dice score for each ground truth to predicted instance
for counter_gt, instance_gt in enumerate(instances_gt):
# create binary mask for current gt instance
gt = mask_gt.copy()
gt[mask_gt != instance_gt] = 0
gt[mask_gt == instance_gt] = 1
masks_row = []
for counter_pred, instance_pred in enumerate(instances_pred):
# make binary mask for current predicted instance
prediction = mask_pred.copy()
prediction[mask_pred != instance_pred] = 0
prediction[mask_pred == instance_pred] = 1
# calculate iou
# show_image(gt, prediction)
iou = compute_iou(gt, prediction)
performance_matrix[counter_gt, counter_pred] = iou
masks_row.append([gt, prediction])
masks.append(masks_row)
# delete all matches smaller than threshold
performance_matrix[performance_matrix < min_iou_for_match] = 0
# assign instrument instances according to hungarian algorithm
label_assignment = hungarian_algorithm(performance_matrix * -1)
label_nr_gt, label_nr_pred = label_assignment
# get performance per instance
-
true_positive_list = []
for i in range(len(label_nr_gt)):
instance_iou = performance_matrix[label_nr_gt[i], label_nr_pred[i]]
true_positive_list.append(instance_iou)
true_positive_list = list(filter(lambda a: a != 0, true_positive_list)) # delete all 0s assigned to a label
true_positive = len(true_positive_list)
false_negative = len(instances_gt) - true_positive
false_positive = len(instances_pred) - true_positive
try:
precision = true_positive / (true_positive + false_positive)
except ZeroDivisionError:
precision = 0
try:
recall = true_positive / (true_positive + false_negative)
except ZeroDivisionError:
recall = 0
output = dict(
true_positive=true_positive,
false_positive=false_positive,
false_negative=false_negative,
precision=precision,
recall=recall
)
return output
-def compute_mean_average_precision(statistic_list):
+def compute_mean_average_precision(statistic_list, plot_curve=False):
"""
Compute the mean average precision:
(https://en.wikipedia.org/wiki/Evaluation_measures_(information_retrieval)#Mean_average_precision)
We define average precision as Area under Curve AUC)
https://medium.com/@jonathan_hui/map-mean-average-precision-for-object-detection-45c121a31173
Args:
statistic_list: 1-dim list, containing statistics dicts (dict definition, see function compute_statistics).
Returns:
the area_under_curve as float
)
"""
# create data frame
data_frame = DataFrame(columns=["true_positive", "false_positive", "false_negative", "precision", "recall"])
# add data
data_frame = data_frame.append(statistic_list)
data_frame = data_frame.reset_index()
# interpolate precision with highest recall for precision
- data_frame = data_frame.sort_values(by="recall", ascending=False)
+ data_frame = data_frame.sort_values(by="recall", ascending=True) # changed from false to true!
precision_interpolated = []
current_highest_value = 0
for index, row in data_frame.iterrows():
if row.precision > current_highest_value:
current_highest_value = row.precision
precision_interpolated.append(current_highest_value)
data_frame['precision_interpolated'] = precision_interpolated
# get changes in interpolated precision curve
data_frame_grouped = data_frame.groupby("recall")
changes = []
for item in data_frame_grouped.groups.items():
current_recall = item[0]
idx_precision = item[1][0]
current_precision_interpolated = data_frame.loc[idx_precision].precision_interpolated
- change = dict(recall=current_recall, precision_interpolated=current_precision_interpolated)
+ current_precision = data_frame.loc[idx_precision].precision
+ change = dict(recall=current_recall, precision_interpolated=current_precision_interpolated, precision=current_precision)
changes.append(change)
# add end and starting point
if changes[0]["recall"] != 0.0:
- changes.insert(0, dict(recall=0, precision_interpolated=changes[0]["precision_interpolated"]))
+ changes.insert(0, dict(recall=0, precision_interpolated=changes[0]["precision_interpolated"], precision=changes[0]["precision"]))
if current_recall < 1:
- changes.append(dict(recall=1, precision_interpolated=current_precision_interpolated))
+ changes.append(dict(recall=1, precision_interpolated=current_precision_interpolated, precision=current_precision))
+
+ if plot_curve:
+ plot_precision = [change['precision'] for change in changes]
+ plot_precision_interpolated = [change['precision_interpolated'] for change in changes]
+ plot_recall = [change['recall'] for change in changes]
+ plt.figure()
+ plt.plot(plot_recall[::-1], plot_precision, '-o')
+ plt.plot(plot_recall[::-1], plot_precision_interpolated, '-o')
+ time.time()
+ plt.savefig(f'/media/rosst/hdd_data/development/robust_mis_2019/precision_recall_curves/{time.time()}.png')
+
# calculate area under curve
area_under_curve = 0
- for i in range(1, len(changes)):
- precision_area = (changes[i]["recall"] - changes[i - 1]["recall"]) * changes[i]["precision_interpolated"]
+ auc_recall = [change['recall'] for change in changes]
+ auc_recall = auc_recall[::-1]
+ auc_precision = [change['precision'] for change in changes]
+ # for i in range(1, len(changes)): # start at one because we use the previous element
+ # precision_area = (changes[i]["recall"] - changes[i - 1]["recall"]) * changes[i]["precision_interpolated"]
+ # area_under_curve += precision_area
+ for i in range(1, len(auc_recall)):
+ precision_area = (auc_recall[i-1]- auc_recall[i]) * auc_precision[i]
area_under_curve += precision_area
-
return area_under_curve
diff --git a/evaluation/test/images/img1/instrument_instances.png b/statistics/test/images/img1/instrument_instances.png
similarity index 100%
rename from evaluation/test/images/img1/instrument_instances.png
rename to statistics/test/images/img1/instrument_instances.png
diff --git a/evaluation/test/images/img1/raw.png b/statistics/test/images/img1/raw.png
similarity index 100%
rename from evaluation/test/images/img1/raw.png
rename to statistics/test/images/img1/raw.png
diff --git a/evaluation/test/images/img2/instrument_instances.png b/statistics/test/images/img2/instrument_instances.png
similarity index 100%
rename from evaluation/test/images/img2/instrument_instances.png
rename to statistics/test/images/img2/instrument_instances.png
diff --git a/evaluation/test/images/img2/raw.png b/statistics/test/images/img2/raw.png
similarity index 100%
rename from evaluation/test/images/img2/raw.png
rename to statistics/test/images/img2/raw.png
diff --git a/evaluation/test/images/img3/instrument_instances.png b/statistics/test/images/img3/instrument_instances.png
similarity index 100%
rename from evaluation/test/images/img3/instrument_instances.png
rename to statistics/test/images/img3/instrument_instances.png
diff --git a/evaluation/test/images/img3/raw.png b/statistics/test/images/img3/raw.png
similarity index 100%
rename from evaluation/test/images/img3/raw.png
rename to statistics/test/images/img3/raw.png
diff --git a/evaluation/test/images/test_map/annotation_1.png b/statistics/test/images/test_map/annotation_1.png
similarity index 100%
rename from evaluation/test/images/test_map/annotation_1.png
rename to statistics/test/images/test_map/annotation_1.png
diff --git a/evaluation/test/images/test_map/annotation_2.png b/statistics/test/images/test_map/annotation_2.png
similarity index 100%
rename from evaluation/test/images/test_map/annotation_2.png
rename to statistics/test/images/test_map/annotation_2.png
diff --git a/evaluation/test/images/test_map/annotation_3.png b/statistics/test/images/test_map/annotation_3.png
similarity index 100%
rename from evaluation/test/images/test_map/annotation_3.png
rename to statistics/test/images/test_map/annotation_3.png
diff --git a/evaluation/test/images/test_map/gt_1.png b/statistics/test/images/test_map/gt_1.png
similarity index 100%
rename from evaluation/test/images/test_map/gt_1.png
rename to statistics/test/images/test_map/gt_1.png
diff --git a/evaluation/test/images/test_map/gt_2.png b/statistics/test/images/test_map/gt_2.png
similarity index 100%
rename from evaluation/test/images/test_map/gt_2.png
rename to statistics/test/images/test_map/gt_2.png
diff --git a/evaluation/test/images/test_map/gt_3.png b/statistics/test/images/test_map/gt_3.png
similarity index 100%
rename from evaluation/test/images/test_map/gt_3.png
rename to statistics/test/images/test_map/gt_3.png
diff --git a/evaluation/test/test_detection_metric.py b/statistics/test/test_detection_metric.py
similarity index 98%
rename from evaluation/test/test_detection_metric.py
rename to statistics/test/test_detection_metric.py
index 2759407..f9b2595 100644
--- a/evaluation/test/test_detection_metric.py
+++ b/statistics/test/test_detection_metric.py
@@ -1,333 +1,333 @@
import unittest
import numpy as np
-from evaluation.mean_average_precision_calculations import compute_mean_average_precision, compute_iou, \
- compute_statistics
+from statistics.mean_average_precision_calculations import compute_iou, compute_statistics, \
+ compute_mean_average_precision
class TestMAPCalculation(unittest.TestCase):
def test_intersection_over_union(self):
# define ground truth
gt_1 = np.array([[0, 0, 0, 0, 0, 0, 0],
[0, 0, 1, 1, 1, 0, 0],
[0, 0, 1, 1, 1, 0, 0],
[0, 0, 1, 1, 1, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0]], np.uint8)
gt_2 = np.array([[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 1, 1, 1, 0, 0],
[0, 0, 1, 1, 1, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0]], np.uint8)
# full intersection
expected_iou_1 = 1
pred_1 = np.array([[0, 0, 0, 0, 0, 0, 0],
[0, 0, 1, 1, 1, 0, 0],
[0, 0, 1, 1, 1, 0, 0],
[0, 0, 1, 1, 1, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0]], np.uint8)
# one missing
expected_iou_2 = 0.8888888
pred_2 = np.array([[0, 0, 0, 0, 0, 0, 0],
[0, 0, 1, 1, 1, 0, 0],
[0, 0, 1, 0, 1, 0, 0],
[0, 0, 1, 1, 1, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0]], np.uint8)
# just one intersected
expected_iou_3 = 0.11111
pred_3 = np.array([[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 1, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0]], np.uint8)
# no intersection
expected_iou_4 = 0
pred_4 = np.array([[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 1, 1, 1],
[0, 0, 0, 0, 1, 1, 1],
[0, 0, 0, 0, 1, 1, 1]], np.uint8)
# empty prediction
expected_iou_5 = 0
pred_5 = np.array([[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0]], np.uint8)
delta = 0.0005
self.assertAlmostEqual(compute_iou(mask_gt=gt_1, mask_pred=pred_1), expected_iou_1, delta=delta)
self.assertAlmostEqual(compute_iou(mask_gt=gt_1, mask_pred=pred_2), expected_iou_2, delta=delta)
self.assertAlmostEqual(compute_iou(mask_gt=gt_1, mask_pred=pred_3), expected_iou_3, delta=delta)
self.assertAlmostEqual(compute_iou(mask_gt=gt_1, mask_pred=pred_4), expected_iou_4, delta=delta)
self.assertAlmostEqual(compute_iou(mask_gt=gt_1, mask_pred=pred_5), expected_iou_5, delta=delta)
self.assertAlmostEqual(compute_iou(mask_gt=gt_2, mask_pred=pred_4), expected_iou_4, delta=delta)
self.assertAlmostEqual(compute_iou(mask_gt=gt_2, mask_pred=pred_5), expected_iou_5, delta=delta)
self.assertTrue(np.isnan(compute_iou(mask_gt=pred_5, mask_pred=pred_5)))
def test_detection_statistics(self):
# define images
img_1 = np.array([[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0]], np.uint8)
img_2 = np.array([[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 1, 1, 0, 0, 0],
[0, 0, 1, 1, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0]], np.uint8)
img_3 = np.array([[1, 1, 0, 0, 0, 0, 0],
[1, 1, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 2, 2, 0],
[0, 0, 0, 0, 2, 2, 0],
[0, 0, 0, 0, 0, 0, 0]], np.uint8)
img_4 = np.array([[1, 1, 0, 0, 0, 0, 0],
[1, 1, 0, 0, 3, 3, 0],
[0, 0, 0, 0, 3, 3, 0],
[0, 4, 4, 0, 0, 0, 0],
[0, 4, 4, 0, 2, 2, 0],
[5, 5, 0, 0, 2, 2, 0],
[5, 5, 0, 0, 0, 0, 0]], np.uint8)
# statistics
# precision = true_positive / (true_positive + false_positive)
# recall = true_positive / (true_positive + false_negative)
result_test_case_1 = compute_statistics(mask_gt=img_1, mask_pred=img_1)
expectation_test_case_1 = dict(
true_positive=0,
false_positive=0,
false_negative=0,
precision=0,
recall=0
)
result_test_case_2 = compute_statistics(mask_gt=img_1, mask_pred=img_2)
expectation_test_case_2 = dict(
true_positive=0,
false_positive=1,
false_negative=0,
precision=0,
recall=0
)
result_test_case_3 = compute_statistics(mask_gt=img_2, mask_pred=img_1)
expectation_test_case_3 = dict(
true_positive=0,
false_positive=0,
false_negative=1,
precision=0,
recall=0
)
result_test_case_4 = compute_statistics(mask_gt=img_3, mask_pred=img_2)
expectation_test_case_4 = dict(
true_positive=0,
false_positive=1,
false_negative=2,
precision=0,
recall=0
)
result_test_case_5 = compute_statistics(mask_gt=img_3, mask_pred=img_4)
expectation_test_case_5 = dict(
true_positive=2,
false_positive=3,
false_negative=0,
precision=2 / (2 + 3),
recall=2 / (2 + 0)
)
result_test_case_6 = compute_statistics(mask_gt=img_4, mask_pred=img_4) # expect fp=0, tp=2, fn=3
expectation_test_case_6 = dict(
true_positive=5,
false_positive=0,
false_negative=0,
precision=1,
recall=1
)
self.assertDictEqual(result_test_case_1, expectation_test_case_1)
self.assertDictEqual(result_test_case_2, expectation_test_case_2)
self.assertDictEqual(result_test_case_3, expectation_test_case_3)
self.assertDictEqual(result_test_case_4, expectation_test_case_4)
self.assertDictEqual(result_test_case_5, expectation_test_case_5)
self.assertDictEqual(result_test_case_6, expectation_test_case_6)
def test_mean_average_precision(self):
statistics_list_1 = [
dict(true_positive=0,
false_positive=0,
false_negative=0,
precision=1.0,
recall=0.2),
dict(true_positive=0,
false_positive=0,
false_negative=0,
precision=1.0,
recall=0.4),
dict(true_positive=0,
false_positive=0,
false_negative=0,
precision=0.67,
recall=0.4),
dict(true_positive=0,
false_positive=0,
false_negative=0,
precision=0.5,
recall=0.4),
dict(true_positive=0,
false_positive=0,
false_negative=0,
precision=0.4,
recall=0.4),
dict(true_positive=0,
false_positive=0,
false_negative=0,
precision=0.5,
recall=0.6),
dict(true_positive=0,
false_positive=0,
false_negative=0,
precision=0.57,
recall=0.8),
dict(true_positive=0,
false_positive=0,
false_negative=0,
precision=0.5,
recall=0.8),
dict(true_positive=0,
false_positive=0,
false_negative=0,
precision=0.44,
recall=0.8),
dict(true_positive=0,
false_positive=0,
false_negative=0,
precision=0.5,
recall=1.0)
]
statistics_list_2 = [
dict(true_positive=0,
false_positive=0,
false_negative=0,
precision=1.0,
recall=0.090909),
dict(true_positive=0,
false_positive=0,
false_negative=0,
precision=0.5,
recall=0.090909),
dict(true_positive=0,
false_positive=0,
false_negative=0,
precision=0.666667,
recall=0.166667),
dict(true_positive=0,
false_positive=0,
false_negative=0,
precision=0.75,
recall=0.230769),
dict(true_positive=0,
false_positive=0,
false_negative=0,
precision=0.6,
recall=0.230769),
dict(true_positive=0,
false_positive=0,
false_negative=0,
precision=0.666667,
recall=0.285714),
dict(true_positive=0,
false_positive=0,
false_negative=0,
precision=0.714286,
recall=0.33333),
dict(true_positive=0,
false_positive=0,
false_negative=0,
precision=0.75,
recall=0.375),
dict(true_positive=0,
false_positive=0,
false_negative=0,
precision=0.66667,
recall=0.375),
dict(true_positive=0,
false_positive=0,
false_negative=0,
precision=0.7,
recall=0.411765),
]
statistics_list_3 = [
dict(true_positive=0,
false_positive=0,
false_negative=0,
precision=1.0,
recall=0.33),
dict(true_positive=0,
false_positive=0,
false_negative=0,
precision=0.5,
recall=0.33),
dict(true_positive=0,
false_positive=0,
false_negative=0,
precision=0.67,
recall=0.67),
dict(true_positive=0,
false_positive=0,
false_negative=0,
precision=0.5,
recall=0.67),
dict(true_positive=0,
false_positive=0,
false_negative=0,
precision=0.4,
recall=0.67),
dict(true_positive=0,
false_positive=0,
false_negative=0,
precision=0.5,
recall=1.0),
dict(true_positive=0,
false_positive=0,
false_negative=0,
precision=0.43,
recall=1.0),
]
delta = 0.0005
self.assertAlmostEqual(compute_mean_average_precision(statistics_list_1), (0.4*1.0+0.4*0.57+0.2*0.5), delta=delta)
self.assertAlmostEqual(compute_mean_average_precision(statistics_list_2), (0.09*1+0.285*0.75+0.625*0.7), delta=delta)
self.assertAlmostEqual(compute_mean_average_precision(statistics_list_3), 0.33*1+0.34*0.67+0.33*0.5, delta=delta)
if __name__ == '__main__':
unittest.main()
\ No newline at end of file
diff --git a/evaluation/test/test_distances.py b/statistics/test/test_distances.py
similarity index 98%
rename from evaluation/test/test_distances.py
rename to statistics/test/test_distances.py
index 3177894..f1da3b5 100644
--- a/evaluation/test/test_distances.py
+++ b/statistics/test/test_distances.py
@@ -1,147 +1,147 @@
import unittest
import numpy as np
from imageio import imread
# single pixels, 2mm away
-from evaluation.dice_calculations import compute_dice_coefficient
-from evaluation.distance_calculations import compute_surface_distances, compute_surface_dice_at_tolerance, \
+from statistics.dice_calculations import compute_dice_coefficient
+from statistics.distance_calculations import compute_surface_distances, compute_surface_dice_at_tolerance, \
compute_average_surface_distance, compute_robust_hausdorff, compute_surface_overlap_at_tolerance
class TestDiceCalculation(unittest.TestCase):
def setUp(self):
self.delta = 0.0005
def test_surface_dice(self):
path = "images\\img{}\\instrument_instances.png".format(3)
# read image
x = imread(path)
# make image binary
x[x < 0.5] = 0
x[x >= 0.5] = 1
mask_gt = np.reshape(x,x.shape + (1,))
surface_distances = compute_surface_distances(mask_gt, mask_gt, (1,1,1))
surface_dice = compute_surface_dice_at_tolerance(surface_distances, 1)
self.assertAlmostEqual(surface_dice, 1.0, delta=self.delta)
def test_single_pixels_2mm_away(self):
mask_gt = np.zeros((128, 128, 128), np.uint8)
mask_pred = np.zeros((128, 128, 128), np.uint8)
mask_gt[50, 60, 70] = 1
mask_pred[50, 60, 72] = 1
surface_distances = compute_surface_distances(mask_gt, mask_pred, spacing_mm=(3, 2, 1))
surface_dice_1mm = compute_surface_dice_at_tolerance(surface_distances, 1)
volumetric_dice = compute_dice_coefficient(mask_gt, mask_pred)
print("surface dice at 1mm: {}".format(surface_dice_1mm))
print("volumetric dice: {}".format(volumetric_dice))
self.assertAlmostEqual(surface_dice_1mm, 0.5, delta=self.delta)
self.assertAlmostEqual(volumetric_dice, 0.0, delta=self.delta)
def test_two_cubes(self):
# two cubes. cube 1 is 100x100x100 mm^3 and cube 2 is 102x100x100 mm^3
mask_gt = np.zeros((100, 100, 100), np.uint8)
mask_pred = np.zeros((100, 100, 100), np.uint8)
spacing_mm = (2, 1, 1)
mask_gt[0:50, :, :] = 1
mask_pred[0:51, :, :] = 1
surface_distances = compute_surface_distances(mask_gt, mask_pred, spacing_mm)
expected_average_distance_gt_to_pred = 0.836145008498
expected_volumetric_dice = 2. * 100 * 100 * 100 / (100 * 100 * 100 + 102 * 100 * 100)
surface_dice_1mm = compute_surface_dice_at_tolerance(surface_distances, 1)
volumetric_dice = compute_dice_coefficient(mask_gt, mask_pred)
print("surface dice at 1mm: {}".format(compute_surface_dice_at_tolerance(surface_distances, 1)))
print("volumetric dice: {}".format(compute_dice_coefficient(mask_gt, mask_pred)))
self.assertAlmostEqual(surface_dice_1mm, expected_average_distance_gt_to_pred, delta=self.delta)
self.assertAlmostEqual(volumetric_dice, expected_volumetric_dice, delta=self.delta)
def test_empty_mask_in_pred(self):
# test empty mask in prediction
mask_gt = np.zeros((128, 128, 128), np.uint8)
mask_pred = np.zeros((128, 128, 128), np.uint8)
mask_gt[50, 60, 70] = 1
# mask_pred[50,60,72] = 1
surface_distances = compute_surface_distances(mask_gt, mask_pred, spacing_mm=(3, 2, 1))
average_surface_distance = compute_average_surface_distance(surface_distances)
hausdorf_100 = compute_robust_hausdorff(surface_distances, 100)
hausdorf_95 = compute_robust_hausdorff(surface_distances, 95)
surface_overlap_1_mm = compute_surface_overlap_at_tolerance(surface_distances, 1)
surface_dice_1mm = compute_surface_dice_at_tolerance(surface_distances, 1)
volumetric_dice = compute_dice_coefficient(mask_gt, mask_pred)
print("average surface distance: {} mm".format(average_surface_distance))
print("hausdorff (100%): {} mm".format(hausdorf_100))
print("hausdorff (95%): {} mm".format(hausdorf_95))
print("surface overlap at 1mm: {}".format(surface_overlap_1_mm))
print("surface dice at 1mm: {}".format(surface_dice_1mm))
print("volumetric dice: {}".format(volumetric_dice))
self.assertAlmostEqual(surface_dice_1mm, 0.0, delta=self.delta)
self.assertAlmostEqual(volumetric_dice, 0.0, delta=self.delta)
def test_empty_mask_in_gt(self):
# test empty mask in ground truth
mask_gt = np.zeros((128, 128, 128), np.uint8)
mask_pred = np.zeros((128, 128, 128), np.uint8)
# mask_gt[50,60,70] = 1
mask_pred[50, 60, 72] = 1
surface_distances = compute_surface_distances(mask_gt, mask_pred, spacing_mm=(3, 2, 1))
average_surface_distance = compute_average_surface_distance(surface_distances)
hausdorf_100 = compute_robust_hausdorff(surface_distances, 100)
hausdorf_95 = compute_robust_hausdorff(surface_distances, 95)
surface_overlap_1_mm = compute_surface_overlap_at_tolerance(surface_distances, 1)
surface_dice_1mm = compute_surface_dice_at_tolerance(surface_distances, 1)
volumetric_dice = compute_dice_coefficient(mask_gt, mask_pred)
print("average surface distance: {} mm".format(average_surface_distance))
print("hausdorff (100%): {} mm".format(hausdorf_100))
print("hausdorff (95%): {} mm".format(hausdorf_95))
print("surface overlap at 1mm: {}".format(surface_overlap_1_mm))
print("surface dice at 1mm: {}".format(surface_dice_1mm))
print("volumetric dice: {}".format(volumetric_dice))
self.assertAlmostEqual(surface_dice_1mm, 0.0, delta=self.delta)
self.assertAlmostEqual(volumetric_dice, 0.0, delta=self.delta)
def test_empty_mask_in_gt_and_pred(self):
# test both masks empty
mask_gt = np.zeros((128, 128, 128), np.uint8)
mask_pred = np.zeros((128, 128, 128), np.uint8)
# mask_gt[50,60,70] = 1
# mask_pred[50,60,72] = 1
surface_distances = compute_surface_distances(mask_gt, mask_pred, spacing_mm=(3, 2, 1))
average_surface_distance = compute_average_surface_distance(surface_distances)
hausdorf_100 = compute_robust_hausdorff(surface_distances, 100)
hausdorf_95 = compute_robust_hausdorff(surface_distances, 95)
surface_overlap_1_mm = compute_surface_overlap_at_tolerance(surface_distances, 1)
surface_dice_1mm = compute_surface_dice_at_tolerance(surface_distances, 1)
volumetric_dice = compute_dice_coefficient(mask_gt, mask_pred)
print("average surface distance: {} mm".format(average_surface_distance))
print("hausdorff (100%): {} mm".format(hausdorf_100))
print("hausdorff (95%): {} mm".format(hausdorf_95))
print("surface overlap at 1mm: {}".format(surface_overlap_1_mm))
print("surface dice at 1mm: {}".format(surface_dice_1mm))
print("volumetric dice: {}".format(volumetric_dice))
self.assertTrue(np.isnan(surface_dice_1mm))
self.assertTrue(np.isnan(volumetric_dice))
if __name__ == '__main__':
unittest.main()
diff --git a/evaluation/test/test_instance_dice.py b/statistics/test/test_instance_dice.py
similarity index 96%
rename from evaluation/test/test_instance_dice.py
rename to statistics/test/test_instance_dice.py
index d764f3b..43b9427 100644
--- a/evaluation/test/test_instance_dice.py
+++ b/statistics/test/test_instance_dice.py
@@ -1,54 +1,55 @@
import unittest
from imageio import imread
-from evaluation.dice_calculations import compute_dice_coefficient, compute_dice_coefficient_per_instance
+
+from statistics.dice_calculations import compute_dice_coefficient, compute_dice_coefficient_per_instance
class TestDiceCalculation(unittest.TestCase):
def test_dice_coefficient(self):
# paths
x_path = "images/img{}/instrument_instances.png".format(1)
y_path = "images/img{}/instrument_instances.png".format(2)
# read images
x = imread(x_path)
y = imread(y_path)
# make images binary
x[x < 0.5] = 0
x[x >= 0.5] = 1
y[y < 0.5] = 0
y[y >= 0.5] = 1
# calculate dice
dice = compute_dice_coefficient(x, y)
# check if correct
expected_dice = 0.011
delta = 0.0005
self.assertAlmostEqual(dice, expected_dice, delta=delta)
def test_multiple_instance_dice_coefficient(self):
# paths
x_path = "images/img{}/instrument_instances.png".format(2)
y_path = "images/img{}/instrument_instances.png".format(3)
# read images
x = imread(x_path)
y = imread(y_path)
# calculate instance dice
instance_dice_scores = compute_dice_coefficient_per_instance(x, y)
# check if correct
expected_dice_scores = dict(background=0.8789, instrument_0=0, instrument_1=0.1676)
delta = 0.0005
for dice_key, expected_dice_key in zip(instance_dice_scores, expected_dice_scores):
dice = instance_dice_scores[dice_key]
expected_dice = expected_dice_scores[expected_dice_key]
self.assertAlmostEqual(dice, expected_dice, delta=delta)
if __name__ == '__main__':
unittest.main()