#!/usr/bin/env python
##############################################################
#
# This script computes statistics to compare models
#
# Usage: python dev/compare_models.py -df path/to/dataframe.csv -n number_of_iterations --test-set
#
# Contributors: Olivier
#
##############################################################
import argparse
import numpy as np
import pandas as pd
from ivadomed import utils as imed_utils
from scipy.stats import ttest_ind_from_stats
from loguru import logger
def get_parser():
parser = argparse.ArgumentParser()
parser.add_argument("-df", "--dataframe", required=True,
help="Path to saved dataframe (csv file).",
metavar=imed_utils.Metavar.file)
parser.add_argument("-n", "--n-iterations", required=True, dest="n_iterations",
type=int, help="Number of times each config was run .",
metavar=imed_utils.Metavar.int)
parser.add_argument("--run_test", dest='run_test', action='store_true',
help="""Evaluate the trained model on the testing sub-set instead of
validation.""")
parser.add_argument("-o", "--output", dest='out', default="comparison_models.csv",
help="if defined will represents the output csv file.",
metavar=imed_utils.Metavar.file)
return parser
[docs]
def compute_statistics(dataframe, n_iterations, run_test=True, csv_out='comparison_models.csv'):
"""Compares the performance of models at inference time on a common testing dataset using paired t-tests.
It uses a dataframe generated by ``scripts/automate_training.py`` with the parameter ``--run-test`` (used to run the
models on the testing dataset). It output dataframes that stores the different statistic (average, std and p_value
between runs). All can be combined and stored in a csv.
.. csv-table:: Example of dataframe
:file: ../../images/df_compare.csv
Usage example::
ivadomed_compare_models -df results.csv -n 2 --run_test
Args:
dataframe (pandas.Dataframe): Dataframe of results generated by automate_training. Flag: ``--dataframe``, ``-df``
n_iterations (int): Indicates the number of time that each experiment (ie set of parameter) was run.
Flag: ``--n_iteration``, ``-n``
run_test (int): Indicates if the comparison is done on the performances on either the testing subdataset (True)
either on the training/validation subdatasets. Flag: ``--run_test``
csv_out (string): Output csv name to store computed value (e.g., df.csv). Default value is model_comparison.csv. Flag ``-o``, ``--output``
"""
avg = dataframe.groupby(['path_output']).mean()
std = dataframe.groupby(['path_output']).std()
logger.info(f"Average dataframe: {avg}")
logger.info(f"Standard deviation dataframe: {std}")
config_logs = list(avg.index.values)
p_values = np.zeros((len(config_logs), len(config_logs)))
i, j = 0, 0
for confA in config_logs:
j = 0
for confB in config_logs:
if run_test:
p_values[i, j] = ttest_ind_from_stats(mean1=avg.loc[confA]["test_dice"],
std1=std.loc[confA]["test_dice"],
nobs1=n_iterations, mean2=avg.loc[confB]["test_dice"],
std2=std.loc[confB]["test_dice"], nobs2=n_iterations).pvalue
else:
p_values[i, j] = ttest_ind_from_stats(mean1=avg.loc[confA]["best_validation_dice"],
std1=std.loc[confA]["best_validation_dice"],
nobs1=n_iterations, mean2=avg.loc[confB]["best_validation_dice"],
std2=std.loc[confB]["best_validation_dice"],
nobs2=n_iterations).pvalue
j += 1
i += 1
p_df = pd.DataFrame(p_values, index=config_logs, columns=config_logs)
logger.info("P-values dataframe")
logger.info(p_df)
if csv_out is not None:
# Unnamed 0 column correspond to run number so we remoe that and add prefix for better readability
df_concat = pd.concat([avg.add_prefix('avg_').drop(['avg_Unnamed: 0'], axis=1),
std.add_prefix('std_').drop(['std_Unnamed: 0'], axis=1), p_df.add_prefix('p-value_')],
axis=1)
df_concat.to_csv(csv_out)
def main(args=None):
imed_utils.init_ivadomed()
parser = get_parser()
args = imed_utils.get_arguments(parser, args)
df = pd.read_csv(args.dataframe)
compute_statistics(df, int(args.n_iterations), bool(args.run_test), args.out)
if __name__ == '__main__':
main()