Source code for ivadomed.scripts.visualize_and_compare_testing_models

###########################################################################################################
#
# This script makes a figure with violinplots and significance values to compare between models
#
#        python3 visualize_and_compare_testing_models.py --ofolders path/to/ofolder1 path/to/ofolder2
#                              --metric metric_to_use --metadata metadata_label string_to_match

# Konstantinos Nasiotis 2021
###########################################################################################################

import matplotlib
from matplotlib import pyplot as plt
import pandas as pd
import numpy as np
import itertools
import seaborn as sns
from scipy.stats import ks_2samp
from ivadomed.utils import init_ivadomed
from pathlib import Path
from loguru import logger
import argparse

matplotlib.rcParams['toolbar'] = 'None'  # Remove buttons

if matplotlib.get_backend() == "agg":
    logger.warning("No backend can be used - Visualization will fail")
else:
    logger.info(f"Using: {matplotlib.get_backend()}  gui")


# ---------------------------------------------------------------------------------------------------------------------#


def get_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument("--ofolders", required=True, nargs="*", dest="ofolders",
                        help="List of log folders from different models.")
    parser.add_argument("--metric", default='dice_class0', nargs=1, type=str, dest="metric",
                        help="Metric from evaluation_3Dmetrics.csv to base the plots on.")
    parser.add_argument("--metadata", required=False, nargs=2, type=str, dest="metadata",
                        help="Selection based on metadata from participants.tsv:"
                             "(1) Label from column (2) string to match")
    return parser


def onclick(event, df):
    # Get the index of the selected violinplot datapoint
    # WARNING: More than one can be selected if they are very close to each other
    #          If that's the case, all will be displayed
    clicked_index = event.ind

    fig = plt.gca()
    # clicking outside the plot area produces a coordinate of None, so we filter those out.
    if None not in clicked_index:
        output_folders = df["EvaluationModel"].unique()
        nfolders = len(output_folders)

        # Remove the previously displayed subject(s)
        # This also takes care of the case where more than one subjects are displayed
        while len(fig.texts) > nfolders + np.math.factorial(nfolders) / (
                np.math.factorial(2) * np.math.factorial(nfolders - 2)):
            fig.texts.pop()

        # This is a hack to find the index of the Violinplot - There should be another way to get this from the
        # figure object
        bins = np.linspace(-1, 1, len(output_folders) + 1)
        i_output_folder = np.where(bins < event.mouseevent.xdata)
        i_output_folder = i_output_folder[-1][0]
        selected_output_folder = df[df["EvaluationModel"] == output_folders[i_output_folder]]

        for iSubject in range(0, len(clicked_index.tolist())):
            frame = plt.text(event.mouseevent.xdata, -0.08 - 0.08 * iSubject + event.mouseevent.ydata,
                             selected_output_folder["subject"][clicked_index[iSubject]], size=10,
                             ha="center", va="center",
                             bbox=dict(facecolor='red', alpha=0.5)
                             )
            # To show the artist
            matplotlib.artist.Artist.set_visible(frame, True)

        plt.show()


[docs] def visualize_and_compare_models(ofolders, metric="dice_class0", metadata=None): """ This function allows violinplots visualization of multiple evaluation models simultaneously and performs a Kolmogorov–Smirnov significance test between each combination of models. The mean values of the datapoints for each violinplot is superimposed on the top. If only one model is selected as input, only the Violinplot will be presented (no test will be superimposed) The datapoints within each violinplot are interactive. The subject_id and MRI sequence of each point are displayed when clicked (as shown on the violinplot to the right of the example figure below). .. note:: If more than 4 model outputs are selected to be compared, the significance tests are not displayed since the figure becomes very busy Usage example:: visualize_and_compare_testing_models.py --ofolders ~/logs/logs_T1w ~/logs/logs_T2w --metric dice_class0 --metadata pathology ms .. image:: ../../images/visualize_and_compare_models.png :width: 600px :align: center Args: ofolders (list): list of folders that contain the outputs of the models to be compared, Flag: ``--ofolders`` metric (str): column of "results_eval/evaluation_3Dmetrics.csv" to be used on the plots (default: dice_class0), Flag: ``--metric`` metadata (list) - Optional: Allows visualization of violinplots only from subjects that match the metadata criteria. 2 elements - (1) column label of the dataframe.csv metadata so only subjects that belong to that category will be used and (2) string to be matched, Flag: ``--metadata``, Example: "--metadata pathology ms" """ # access CLI options logger.debug(f"ofolders: {ofolders}") logger.debug(f"metric: {metric}") if metadata is None: metadata = [] if metadata: logger.debug(f"metadata: {metadata}") # Do a quick check that all the required files are present for folder in ofolders: if not Path(folder, 'results_eval', 'evaluation_3Dmetrics.csv').exists(): logger.error(f"evaluation_3Dmetrics.csv file is not present within {Path(folder, 'results_eval')}") raise Exception('evaluation_3Dmetrics.csv missing') if not Path(folder, 'bids_dataframe.csv').exists(): logger.error(f"bids_dataframe.csv file is not present within {folder}") raise Exception('bids_dataframe.csv missing') if len(ofolders) < 1: raise Exception('No folders were selected - Nothing to show') np_lst = [] for folder in ofolders: result = pd.read_csv(str(Path(folder, 'results_eval', 'evaluation_3Dmetrics.csv'))) if metadata: participant_metadata = pd.read_table(str(Path(folder, 'bids_dataframe.csv')), sep=',') # Select only the subjects that satisfy the --metadata input selected_subjects = participant_metadata[participant_metadata[metadata[0]] == metadata[1]][ "filename"].tolist() selected_subjects = [i.replace(".nii.gz", "") for i in selected_subjects] # Now select only the scores from these subjects result_subject_ids = result["image_id"] result = result.iloc[ [i for i in range(len(result_subject_ids)) if result_subject_ids[i] in selected_subjects]] if result.empty: logger.warning(f"No subject meet the selected criteria - skipping plot for: {folder}") if not result.empty: scores = result[metric] folders = [Path(folder).resolve().name] * len(scores) subject_id = result["image_id"] combined = np.column_stack((folders, scores.astype(np.object, folders), subject_id)) np_lst.append(combined) columnNames = ["EvaluationModel", metric, 'subject'] rows = np.vstack(np_lst) df = pd.DataFrame(rows, columns=columnNames) nFolders = len(ofolders) combinedNumbers = list(itertools.combinations(range(nFolders), 2)) combinedFolders = list(itertools.combinations(ofolders, 2)) # Pandas annoying issues df[metric] = df[metric].astype('float64') if not df.empty: # Plot all violinplots plt1 = sns.violinplot(x="EvaluationModel", y=metric, data=df, palette="muted", saturation=0.3, inner='quartile', picker=True, pickradius=1) plt2 = sns.stripplot(x="EvaluationModel", y=metric, data=df, linewidth=0.5, edgecolor="black", jitter=True, zorder=1, picker=True, pickradius=1) # Display the mean performance on top of every violinplot for i in range(len(ofolders)): # This will be used to plot the mean value on top of each individual violinplot temp = df[metric][df['EvaluationModel'] == Path(ofolders[i]).resolve().name] plt.text(i, df[metric].max() + 0.07, str((100 * temp.mean()).round() / 100), ha='center', va='top', color='r', picker=True) if len(ofolders) > 1 and len(ofolders) < 5: # Perform a Kolmogorov–Smirnov test for all combinations of results & connect the corresponding Violinplots for i in range(len(combinedNumbers)): dataX = df[metric][df['EvaluationModel'] == Path(combinedFolders[i][0]).resolve().name] dataY = df[metric][df['EvaluationModel'] == Path(combinedFolders[i][1]).resolve().name] ks_test = ks_2samp(dataX, dataY) x1, x2 = combinedNumbers[i] y, h, col = df[metric].min() - 0.06 - 0.03 * i, -0.01, 'k' plt.plot([x1, x1, x2, x2], [y, y + h, y + h, y], lw=1.5, c=col, picker=True) # Show if the differentiation of the distributions is : # Not significant: ns, significant: *, very significant: *** if ks_test.pvalue >= 0.5: plt.text((x1 + x2) * .5, y + h, "ns", ha='center', va='bottom', color=col, picker=True) elif 0.5 > ks_test.pvalue >= 0.01: plt.text((x1 + x2) * .5, y + h, "*", ha='center', va='bottom', color='r', picker=True) elif ks_test.pvalue < 0.01: plt.text((x1 + x2) * .5, y + h, "***", ha='center', va='bottom', color='r', picker=True) if metadata: plt.title("Metric: " + metric + "\nMetadata: " + metadata[0] + ":" + metadata[1], picker=True) else: plt.title("Metric: " + metric, picker=True) plt.grid() plt.text(0, 0, " ") # One empty entry is introduced here so it is popped in the function plt.gca().figure.canvas.mpl_connect('pick_event', lambda event: onclick(event, df)) plt.show(block=True) else: logger.warning("No subjects meet the criteria selected for any model. " "Probably you need to change the --metadata / --metric selection")
def main(): init_ivadomed() parser = get_parser() args = parser.parse_args() # Run automate training visualize_and_compare_models(args.ofolders, args.metric, args.metadata) if __name__ == '__main__': main()